[CPU][Zen] Add DA8W4 (W4A8) int4 support for dense and MoE layers (#54024)

Signed-off-by: R <Ganesh.R@amd.com>
Signed-off-by: Ganesh R <Ganesh.R@amd.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Ganesh R
2026-10-01 15:18:25 +08:00
committed by GitHub
co-authored by Cursor mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 4d6874d137
commit e12291d733
9 changed files with 1184 additions and 15 deletions
+4
View File
@@ -24,6 +24,9 @@ steps:
- tests/kernels/test_onednn.py
- tests/kernels/test_awq_int4_to_int8.py
- tests/kernels/quantization/test_cpu_fp8_scaled_mm.py
- tests/kernels/quantization/test_zen_da8w4.py
- vllm/model_executor/kernels/linear/mixed_precision/zentorch.py
- vllm/model_executor/layers/fused_moe/experts/zentorch_moe.py
- tests/kernels/mamba/cpu/test_cpu_gdn_ops.py
- tests/kernels/mamba/test_cpu_short_conv.py
- tests/kernels/mamba/test_causal_conv1d.py
@@ -46,6 +49,7 @@ steps:
tests/kernels/test_onednn.py \
tests/kernels/test_awq_int4_to_int8.py \
tests/kernels/quantization/test_cpu_fp8_scaled_mm.py \
tests/kernels/quantization/test_zen_da8w4.py \
tests/kernels/mamba/cpu/test_cpu_gdn_ops.py \
tests/kernels/mamba/test_causal_conv1d.py \
tests/kernels/mamba/test_mamba_ssm.py \
@@ -0,0 +1,661 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for the zentorch DA8W4 (W4A8) int4 linear and MoE paths on Zen CPUs.
The zentorch ops are mocked with reference implementations when zentorch is not
installed, so the layout and dispatch contracts are covered in CI.
"""
import dataclasses
import pytest
import torch
from compressed_tensors.compressors.pack_quantized.helpers import pack_to_int32
from tests.kernels.quant_utils import ref_dynamic_per_token_quant
from vllm.model_executor.kernels.linear.mixed_precision.MPLinearKernel import (
MPLinearLayerConfig,
)
from vllm.model_executor.kernels.linear.mixed_precision.zentorch import (
ZentorchWNA16LinearKernel,
_import_unpack_from_int32,
)
from vllm.model_executor.layers.fused_moe.oracle import int_wna16
from vllm.model_executor.layers.fused_moe.oracle.int_wna16 import WNA16MoEBackend
from vllm.platforms import current_platform
from vllm.scalar_type import scalar_types
if not current_platform.is_cpu():
pytest.skip("skipping CPU-only tests", allow_module_level=True)
GROUP_SIZE = 128
IN_FEATURES = 512
OUT_FEATURES = 128
def _unpack_s4(packed: torch.Tensor, in_features: int) -> torch.Tensor:
"""int8 [N, K/2] (or int32 [N, K/8]) packed s4 -> float32 [N, K]."""
words = packed.view(torch.int32)
out = torch.zeros(words.shape[0], in_features, dtype=torch.float32)
for i in range(8):
nibble = (words >> (4 * i)) & 0xF
out[:, i::8] = torch.where(nibble > 7, nibble - 16, nibble).float()
return out
def _repack_s4(unpacked: torch.Tensor) -> torch.Tensor:
"""Reference zentorch_woq_repack_weight: int8 [N, K] -> int32 [N, K/8]."""
n, k = unpacked.shape
values = unpacked.to(torch.int32).reshape(n, k // 8, 8)
out = torch.zeros(n, k // 8, dtype=torch.int32)
for i in range(8):
out |= (values[:, :, i] & 0xF) << (4 * i)
return out
def _quantize_per_group(
weight: torch.Tensor, group_size: int
) -> tuple[torch.Tensor, torch.Tensor]:
"""Symmetric int4 per-group quantization -> (int8 in [-8, 7], scale [N, G])."""
out_features, in_features = weight.shape
grouped = weight.float().reshape(
out_features, in_features // group_size, group_size
)
scale = grouped.abs().amax(dim=-1) / 8.0
quantized = (grouped / scale.unsqueeze(-1)).round().clamp(-8, 7).to(torch.int8)
return quantized.reshape(out_features, in_features), scale.to(torch.bfloat16)
def _dequantize_per_group(
quantized: torch.Tensor, scale: torch.Tensor, group_size: int
) -> torch.Tensor:
"""Dequantize with [N, G] scales into a float32 [N, K] weight."""
return quantized.float() * scale.float().repeat_interleave(group_size, dim=1)
def _dynamic_quant_matmul(
x: torch.Tensor, weight: torch.Tensor, bias: torch.Tensor | None
) -> torch.Tensor:
"""DA8W4 reference: per-token int8 activation quant against a bf16 weight."""
x_q, scale = ref_dynamic_per_token_quant(x.float(), torch.int8)
out = (x_q.float() @ weight.t()) * scale
if bias is not None:
out = out + bias.float()
return out
@pytest.fixture
def mock_zentorch_ops():
"""Register reference zentorch ops when zentorch is not installed."""
if hasattr(torch.ops.zentorch, "zentorch_dynamic_qlinear"):
yield None
return
calls: dict[str, tuple] = {}
def _dynamic_qlinear(input, weight, weight_scales, bias=None, zentorch_op_name=""):
calls["dynamic_qlinear"] = (input, weight, weight_scales, bias)
in_features = weight.view(torch.int32).shape[1] * 8
group_size = in_features // weight_scales.shape[0]
# weight_scales is [G, N]; expand to a [N, K] dequantized weight.
scales = weight_scales.float().t().repeat_interleave(group_size, dim=1)
weight_deq = _unpack_s4(weight, in_features) * scales
out = _dynamic_quant_matmul(input, weight_deq, bias)
return out.to(input.dtype)
def _fused_moe(
output,
input,
w13,
w2,
w13_bias,
w2_bias,
topk_weights,
topk_id,
skip_weighted,
act,
w13_scales=None,
w2_scales=None,
zentorch_op_name="",
):
calls["fused_moe"] = (
input,
w13,
w2,
topk_weights,
topk_id,
skip_weighted,
act,
w13_scales,
w2_scales,
)
output.zero_()
lib_def = torch.library.Library("zentorch", "DEF")
lib_def.define("zentorch_woq_repack_weight(Tensor unpacked_weight) -> Tensor")
lib_def.define(
"zentorch_dynamic_qlinear(Tensor input, Tensor weight, "
"Tensor weight_scales, Tensor? bias=None, *, "
"str zentorch_op_name='zentorch::zentorch_dynamic_qlinear') -> Tensor"
)
lib_def.define(
"zentorch_fused_moe(Tensor(a!) output, Tensor input, Tensor w13, "
"Tensor w2, Tensor? w13_bias, Tensor? w2_bias, Tensor topk_weights, "
"Tensor topk_id, bool skip_weighted, str act, "
"Tensor? w13_scales=None, Tensor? w2_scales=None, *, "
"str zentorch_op_name='zentorch::zentorch_fused_moe') -> ()"
)
lib_impl = torch.library.Library("zentorch", "IMPL", "CPU")
lib_impl.impl("zentorch_woq_repack_weight", _repack_s4)
lib_impl.impl("zentorch_dynamic_qlinear", _dynamic_qlinear)
lib_impl.impl("zentorch_fused_moe", _fused_moe)
yield calls
lib_impl._destroy()
lib_def._destroy()
def _make_config(
act_type: torch.dtype = torch.bfloat16,
weight_type=scalar_types.uint4b8,
zero_points: bool = False,
group_size: int = GROUP_SIZE,
) -> MPLinearLayerConfig:
return MPLinearLayerConfig(
full_weight_shape=(IN_FEATURES, OUT_FEATURES),
partition_weight_shape=(IN_FEATURES, OUT_FEATURES),
weight_type=weight_type,
act_type=act_type,
group_size=group_size,
zero_points=zero_points,
)
def _make_layer(group_size: int = GROUP_SIZE) -> tuple[torch.nn.Module, torch.Tensor]:
"""Build a layer holding a CT-packed int4 weight, plus its dequantized form."""
torch.manual_seed(0)
weight = torch.randn(OUT_FEATURES, IN_FEATURES, dtype=torch.bfloat16)
quantized, scale = _quantize_per_group(weight, group_size)
# compressed-tensors stores [N, K//8] int32, packed along the input dim.
packed = pack_to_int32(quantized, 4, packed_dim=1)
layer = torch.nn.Module()
layer.weight_packed = torch.nn.Parameter(packed, requires_grad=False)
layer.weight_packed.input_dim = 1
layer.weight_packed.packed_dim = 1
layer.weight_scale = torch.nn.Parameter(scale, requires_grad=False)
layer.weight_zero_point = None
return layer, _dequantize_per_group(quantized, scale, group_size)
def _make_kernel(config: MPLinearLayerConfig) -> ZentorchWNA16LinearKernel:
return ZentorchWNA16LinearKernel(
config, "weight_packed", "weight_scale", "weight_zero_point"
)
# ---------------------------------------------------------------------------
# Dense DA8W4
# ---------------------------------------------------------------------------
def test_da8w4_eligible_for_symmetric_bf16_layer(mock_zentorch_ops, monkeypatch):
monkeypatch.setattr(
"vllm.model_executor.kernels.linear.mixed_precision.zentorch."
"current_platform.is_zen_cpu",
lambda: True,
)
layer, _ = _make_layer()
kernel = _make_kernel(_make_config())
assert kernel._maybe_process_da8w4_weights(layer)
@pytest.mark.parametrize(
"config_kwargs,reason",
[
({"act_type": torch.float32}, "f32 activations are rejected"),
(
{"weight_type": scalar_types.uint4, "zero_points": True},
"asymmetric int4 is unsupported",
),
],
)
def test_da8w4_not_eligible(mock_zentorch_ops, monkeypatch, config_kwargs, reason):
monkeypatch.setattr(
"vllm.model_executor.kernels.linear.mixed_precision.zentorch."
"current_platform.is_zen_cpu",
lambda: True,
)
layer, _ = _make_layer()
kernel = _make_kernel(_make_config(**config_kwargs))
assert not kernel._maybe_process_da8w4_weights(layer), reason
def test_da8w4_not_eligible_when_env_disabled(mock_zentorch_ops, monkeypatch):
monkeypatch.setattr(
"vllm.model_executor.kernels.linear.mixed_precision.zentorch."
"current_platform.is_zen_cpu",
lambda: True,
)
monkeypatch.setattr(
"vllm.model_executor.kernels.linear.mixed_precision.zentorch."
"envs.VLLM_CPU_INT4_W4A8",
False,
)
layer, _ = _make_layer()
kernel = _make_kernel(_make_config())
assert not kernel._maybe_process_da8w4_weights(layer)
def test_da8w4_process_weights_layout(mock_zentorch_ops, monkeypatch):
monkeypatch.setattr(
"vllm.model_executor.kernels.linear.mixed_precision.zentorch."
"current_platform.is_zen_cpu",
lambda: True,
)
layer, _ = _make_layer()
kernel = _make_kernel(_make_config())
kernel.process_weights_after_loading(layer)
assert layer._zentorch_da8w4
assert layer._zentorch_kind == "compressed_tensors_w4a8_da8w4"
# Packed s4 holds two nibbles per byte, and scales transpose to {G, N}.
assert layer._zentorch_da8w4_packed.dtype == torch.int8
assert layer._zentorch_da8w4_packed.shape == (OUT_FEATURES, IN_FEATURES // 2)
assert layer._zentorch_da8w4_scale.dtype == torch.bfloat16
assert layer._zentorch_da8w4_scale.shape == (
IN_FEATURES // GROUP_SIZE,
OUT_FEATURES,
)
assert layer._zentorch_da8w4_packed.is_contiguous()
assert layer._zentorch_da8w4_scale.is_contiguous()
# The checkpoint parameters are released once repacked.
assert layer.weight_packed.numel() == 0
assert layer.weight_scale.numel() == 0
def test_da8w4_apply_weights_matches_dequantized_reference(
mock_zentorch_ops, monkeypatch
):
monkeypatch.setattr(
"vllm.model_executor.kernels.linear.mixed_precision.zentorch."
"current_platform.is_zen_cpu",
lambda: True,
)
layer, weight_deq = _make_layer()
kernel = _make_kernel(_make_config())
kernel.process_weights_after_loading(layer)
x = torch.randn(4, IN_FEATURES, dtype=torch.bfloat16)
bias = torch.randn(OUT_FEATURES, dtype=torch.bfloat16)
out = kernel.apply_weights(layer, x, bias)
expected = _dynamic_quant_matmul(x, weight_deq, bias)
assert out.dtype == torch.bfloat16
torch.testing.assert_close(out.float(), expected, rtol=2e-2, atol=2e-2)
def test_da8w4_falls_back_to_w4a16_when_op_missing(monkeypatch):
"""Without zentorch_dynamic_qlinear the layer must not take the DA8W4 path."""
monkeypatch.setattr(
"vllm.model_executor.kernels.linear.mixed_precision.zentorch.has_zentorch_op",
lambda ops: "zentorch_dynamic_qlinear" not in ops,
)
layer, _ = _make_layer()
kernel = _make_kernel(_make_config())
assert not kernel._maybe_process_da8w4_weights(layer)
def test_can_implement_requires_zen_cpu(monkeypatch):
monkeypatch.setattr(
"vllm.model_executor.kernels.linear.mixed_precision.zentorch."
"current_platform.is_zen_cpu",
lambda: False,
)
ok, reason = ZentorchWNA16LinearKernel.can_implement(_make_config())
assert not ok
assert reason is not None
def test_can_implement_accepts_shapes_cpuwna16_rejects(monkeypatch):
"""CPUWNA16's N/K % 32 rule is a oneDNN packing constraint zentorch lacks."""
from vllm.model_executor.kernels.linear.mixed_precision.cpu import (
CPUWNA16LinearKernel,
)
monkeypatch.setattr(
"vllm.model_executor.kernels.linear.mixed_precision.zentorch."
"current_platform.is_zen_cpu",
lambda: True,
)
monkeypatch.setattr(
"vllm.model_executor.kernels.linear.mixed_precision.cpu."
"current_platform.is_cpu",
lambda: True,
)
monkeypatch.setattr(
"vllm.model_executor.kernels.linear.mixed_precision.zentorch.has_zentorch_op",
lambda ops: True,
)
unaligned = dataclasses.replace(
_make_config(), partition_weight_shape=(IN_FEATURES, OUT_FEATURES - 16)
)
assert unaligned.partition_weight_shape[1] % 32 != 0
ok, _ = CPUWNA16LinearKernel.can_implement(unaligned)
assert not ok, "the parent is expected to turn this shape away"
ok, reason = ZentorchWNA16LinearKernel.can_implement(unaligned)
assert ok, reason
@pytest.mark.parametrize(
"group_size,expected_ok",
[
(-1, True), # per-channel, one group spanning K
(GROUP_SIZE, True),
(IN_FEATURES, True),
(6, False), # even, so the parent's multiple-of-2 rule would allow it
(0, False),
],
)
def test_can_implement_group_size_must_divide_k(monkeypatch, group_size, expected_ok):
"""ZenDNN WOQ groups along K, so a group that straddles the end is unusable."""
monkeypatch.setattr(
"vllm.model_executor.kernels.linear.mixed_precision.zentorch."
"current_platform.is_zen_cpu",
lambda: True,
)
monkeypatch.setattr(
"vllm.model_executor.kernels.linear.mixed_precision.zentorch.has_zentorch_op",
lambda ops: True,
)
ok, reason = ZentorchWNA16LinearKernel.can_implement(
_make_config(group_size=group_size)
)
assert ok is expected_ok, reason
if not expected_ok:
assert reason is not None
assert "must divide input size" in reason
# ---------------------------------------------------------------------------
# MoE DA8W4
# ---------------------------------------------------------------------------
NUM_EXPERTS = 4
HIDDEN = 256
INTERMEDIATE = 128
def _make_moe_weights(group_size: int = 32):
"""Build CT-layout MoE weights: w13 [E, 2I, H//8], w2 [E, H, I//8]."""
torch.manual_seed(0)
w13_q, w13_s, w2_q, w2_s = [], [], [], []
for _ in range(NUM_EXPERTS):
w13 = torch.randn(2 * INTERMEDIATE, HIDDEN, dtype=torch.bfloat16)
q13, s13 = _quantize_per_group(w13, group_size)
# CT is N-first and packs along the input dim: [N, K//8] per expert.
w13_q.append(pack_to_int32(q13, 4, packed_dim=1))
w13_s.append(s13)
w2 = torch.randn(HIDDEN, INTERMEDIATE, dtype=torch.bfloat16)
q2, s2 = _quantize_per_group(w2, group_size)
w2_q.append(pack_to_int32(q2, 4, packed_dim=1))
w2_s.append(s2)
return (
torch.stack(w13_q),
torch.stack(w2_q),
torch.stack(w13_s),
torch.stack(w2_s),
)
def test_zen_cpu_first_in_cpu_backend_priority(monkeypatch):
monkeypatch.setattr(int_wna16.current_platform, "is_cpu", lambda: True)
backends = int_wna16._get_priority_backends()
assert backends[0] == WNA16MoEBackend.ZEN_CPU
# The generic CPU backend stays available as a fallback.
assert WNA16MoEBackend.CPU in backends
def test_zen_cpu_process_weights_layout(mock_zentorch_ops):
group_size = 32
w13, w2, w13_scale, w2_scale = _make_moe_weights(group_size)
assert w13.shape == (NUM_EXPERTS, 2 * INTERMEDIATE, HIDDEN // 8)
assert w2.shape == (NUM_EXPERTS, HIDDEN, INTERMEDIATE // 8)
converted = int_wna16._process_weights_zen_cpu(w13, w2, w13_scale, w2_scale)
w13_out, w2_out, w13_s_out, w2_s_out = converted[:4]
# zentorch consumes [E, N, K/8] packed s4 with per-group [E, G, N] scales.
assert w13_out.shape == (NUM_EXPERTS, 2 * INTERMEDIATE, HIDDEN // 8)
assert w2_out.shape == (NUM_EXPERTS, HIDDEN, INTERMEDIATE // 8)
assert w13_out.dtype == w2_out.dtype == torch.int32
assert w13_s_out.shape == (NUM_EXPERTS, HIDDEN // group_size, 2 * INTERMEDIATE)
assert w2_s_out.shape == (NUM_EXPERTS, INTERMEDIATE // group_size, HIDDEN)
assert w13_s_out.dtype == w2_s_out.dtype == torch.bfloat16
# Symmetric checkpoints carry no zero points into the kernel.
assert converted[8] is None and converted[9] is None
def test_zen_cpu_repack_is_value_exact(mock_zentorch_ops):
"""The repacked weight must dequantize back to the checkpoint values."""
group_size = 32
w13, w2, w13_scale, w2_scale = _make_moe_weights(group_size)
w13_out = int_wna16._process_weights_zen_cpu(w13, w2, w13_scale, w2_scale)[0]
expected = _import_unpack_from_int32()(
w13,
4,
torch.Size([NUM_EXPERTS, 2 * INTERMEDIATE, HIDDEN]),
packed_dim=1,
)
for expert in range(NUM_EXPERTS):
torch.testing.assert_close(
_unpack_s4(w13_out[expert], HIDDEN),
expected[expert].float(),
)
@pytest.mark.parametrize(
"group_size,expected_ok",
[(32, True), (128, True), (-1, False), (2, False)],
)
def test_zen_cpu_group_size_gating(group_size, expected_ok):
quant_config = type("Args", (), {"group_size": group_size})()
reason = int_wna16._backend_incompatibility_reason(
WNA16MoEBackend.ZEN_CPU,
moe_config=None,
quant_config=quant_config,
may_have_zp=False,
may_have_bias=False,
allow_tile_padding=True,
)
assert (reason is None) == expected_ok
@pytest.mark.parametrize(
"may_have_zp,may_have_bias,expected_ok",
[(True, False, False), (False, True, True), (True, True, False)],
)
def test_zen_cpu_rejects_zero_points_but_takes_bias(
may_have_zp, may_have_bias, expected_ok
):
quant_config = type("Args", (), {"group_size": 128})()
reason = int_wna16._backend_incompatibility_reason(
WNA16MoEBackend.ZEN_CPU,
moe_config=None,
quant_config=quant_config,
may_have_zp=may_have_zp,
may_have_bias=may_have_bias,
allow_tile_padding=True,
)
assert (reason is None) == expected_ok
def test_zen_cpu_rejects_moe_wna16_layout():
from vllm.model_executor.layers.quantization.moe_wna16 import MoeWNA16Config
quant_config = MoeWNA16Config(
linear_quant_method="gptq",
weight_bits=4,
group_size=128,
has_zp=False,
lm_head_quantized=False,
modules_to_not_convert=None,
full_config={},
)
reason = int_wna16._backend_incompatibility_reason(
WNA16MoEBackend.ZEN_CPU,
moe_config=None,
quant_config=quant_config,
may_have_zp=False,
may_have_bias=False,
allow_tile_padding=True,
)
assert reason is not None
def test_zen_cpu_disabled_by_env(monkeypatch):
monkeypatch.setattr(int_wna16.envs, "VLLM_CPU_INT4_W4A8", False)
quant_config = type("Args", (), {"group_size": 128})()
reason = int_wna16._backend_incompatibility_reason(
WNA16MoEBackend.ZEN_CPU,
moe_config=None,
quant_config=quant_config,
may_have_zp=False,
may_have_bias=False,
allow_tile_padding=True,
)
assert reason is not None
def test_zen_cpu_backend_maps_to_experts_class():
from vllm.model_executor.layers.fused_moe.experts.zentorch_moe import (
ZentorchExpertsInt4,
)
assert int_wna16.backend_to_kernel_cls(WNA16MoEBackend.ZEN_CPU) == [
ZentorchExpertsInt4
]
def test_zen_experts_support_predicates():
from vllm.model_executor.kernels.linear.zentorch_utils import (
_ZENTORCH_MOE_ACTIVATIONS,
)
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.experts.zentorch_moe import (
ZentorchExpertsInt4,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
kInt4Static,
kInt4Static32,
kInt8StaticChannelSym,
)
assert ZentorchExpertsInt4._supports_quant_scheme(kInt4Static, None)
assert ZentorchExpertsInt4._supports_quant_scheme(kInt4Static32, None)
assert not ZentorchExpertsInt4._supports_quant_scheme(kInt8StaticChannelSym, None)
for act in MoEActivation:
expected = act.value in _ZENTORCH_MOE_ACTIVATIONS
assert ZentorchExpertsInt4._supports_activation(act) == expected
assert ZentorchExpertsInt4.requires_interleaved_w13
assert not ZentorchExpertsInt4._supports_no_act_and_mul()
assert ZentorchExpertsInt4._supports_parallel_config(
type("Par", (), {"use_ep": False})()
)
assert not ZentorchExpertsInt4._supports_parallel_config(
type("Par", (), {"use_ep": True})()
)
def test_zen_experts_take_custom_routing():
"""Models with their own router (gemma-4) reach select_experts through the
config captured off the layer, since apply() cannot be handed the callable."""
from types import SimpleNamespace
from tests.kernels.moe.test_cpu_fused_moe import _StubMoELayer
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.config import RoutingMethodType
from vllm.model_executor.layers.fused_moe.experts.zentorch_moe import (
ZentorchExpertsInt4,
)
assert ZentorchExpertsInt4._supports_routing_method(
RoutingMethodType.Custom, None, None
)
def routing_fn(**kwargs):
raise AssertionError("not called")
layer = _StubMoELayer(
torch.zeros(NUM_EXPERTS, 2 * INTERMEDIATE, HIDDEN),
torch.zeros(NUM_EXPERTS, HIDDEN, INTERMEDIATE),
MoEActivation.SILU,
)
layer.renormalize = True
layer.custom_routing_function = routing_fn
experts = SimpleNamespace(renormalize=False, custom_routing_function=None)
ZentorchExpertsInt4.process_weights_after_loading(experts, layer)
assert experts.custom_routing_function is routing_fn
assert experts.renormalize is True
def _swigluoai_perm(activation, experts_cls=None):
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import (
swigluoai_w13_interleave_perm,
)
from vllm.model_executor.layers.fused_moe.experts.zentorch_moe import (
ZentorchExpertsInt4,
)
return swigluoai_w13_interleave_perm(
experts_cls or ZentorchExpertsInt4,
activation,
2 * INTERMEDIATE,
torch.device("cpu"),
)
def test_swigluoai_perm_only_for_zen_swigluoai():
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
assert _swigluoai_perm(MoEActivation.SWIGLUOAI) is not None
# Other activations keep the half-split layout the loader leaves.
assert _swigluoai_perm(MoEActivation.SILU) is None
# object stands in for any kernel that never set requires_interleaved_w13.
assert _swigluoai_perm(MoEActivation.SWIGLUOAI, experts_cls=object) is None
def test_swigluoai_perm_interleaves_weights_scales_and_bias(mock_zentorch_ops):
"""The permutation is a gather on w13's output-channel axis, which is dim 1
for the N-first packed weights and their group scales."""
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
group_size = 32
w13, w2, w13_scale, w2_scale = _make_moe_weights(group_size)
bias = torch.randn(NUM_EXPERTS, 2 * INTERMEDIATE, dtype=torch.bfloat16)
perm = _swigluoai_perm(MoEActivation.SWIGLUOAI)
baseline = int_wna16._process_weights_zen_cpu(w13, w2, w13_scale, w2_scale)[0]
permuted = int_wna16._process_weights_zen_cpu(
w13[:, perm].contiguous(), w2, w13_scale[:, perm].contiguous(), w2_scale
)[0]
for expert in range(NUM_EXPERTS):
before = _unpack_s4(baseline[expert], HIDDEN)
after = _unpack_s4(permuted[expert], HIDDEN)
torch.testing.assert_close(after[0::2], before[:INTERMEDIATE])
torch.testing.assert_close(after[1::2], before[INTERMEDIATE:])
torch.testing.assert_close(bias[:, perm][:, 0::2], bias[:, :INTERMEDIATE])
torch.testing.assert_close(bias[:, perm][:, 1::2], bias[:, INTERMEDIATE:])
+2 -1
View File
@@ -912,7 +912,8 @@ environment_variables: dict[str, Callable[[], Any]] = {
"VLLM_ZENTORCH_WEIGHT_PREPACK": lambda: bool(
int(os.getenv("VLLM_ZENTORCH_WEIGHT_PREPACK", "1"))
),
# (CPU backend only) whether to use SGLang INT4 W4A8 kernels for AWQ.
# (CPU backend only) whether to use SGLang INT4 W4A8 kernels for AWQ, and
# on Zen CPUs whether to serve int4 checkpoints as DA8W4 rather than W4A16.
"VLLM_CPU_INT4_W4A8": lambda: bool(int(os.getenv("VLLM_CPU_INT4_W4A8", "1"))),
# If the env var is set, Ray Compiled Graph uses the specified
# channel type to communicate between workers belonging to
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Zentorch W4A16 GPTQ weight-only-quantized linear kernel for AMD Zen CPUs.
"""Zentorch int4 weight-quantized linear kernels for AMD Zen CPUs.
Selected by ``choose_mp_linear_kernel`` ahead of the generic oneDNN-backed
``CPUWNA16LinearKernel``. When ``can_implement`` rejects a layer, the selector
@@ -9,12 +9,13 @@ falls through to the next kernel in ``_POSSIBLE_KERNELS[PlatformEnum.CPU]``.
import torch
from vllm import envs
from vllm.logger import init_logger
from vllm.model_executor.kernels.linear.zentorch_utils import has_zentorch_op
from vllm.platforms import current_platform
from vllm.scalar_type import scalar_types
from .cpu import CPUWNA16LinearKernel
from .cpu import _CPUWNA16_SUPPORTED_QUANT_TYPES, CPUWNA16LinearKernel
from .MPLinearKernel import MPLinearLayerConfig
logger = init_logger(__name__)
@@ -34,17 +35,35 @@ def _import_unpack_from_int32():
class ZentorchWNA16LinearKernel(CPUWNA16LinearKernel):
"""W4A16 GPTQ kernel backed by ``torch.ops.zentorch.zentorch_woq_linear``."""
"""Int4 kernel backed by ``zentorch_dynamic_qlinear`` (DA8W4) or
``zentorch_woq_linear`` (W4A16)."""
@classmethod
def can_implement(cls, c: MPLinearLayerConfig) -> tuple[bool, str | None]:
ok, reason = super().can_implement(c)
if not ok:
return ok, reason
if not current_platform.is_zen_cpu():
return False, "ZentorchWNA16 requires an AMD Zen CPU."
# Not super(): its N/K multiple-of-32 rule is a oneDNN packing
# constraint the zentorch repack does not share.
if c.weight_type not in _CPUWNA16_SUPPORTED_QUANT_TYPES:
return (
False,
f"Quant type ({c.weight_type}) not supported by "
"CPUWNA16, supported types are: "
f"{_CPUWNA16_SUPPORTED_QUANT_TYPES}",
)
# -1 is per-channel, one group spanning K. Anything else has to divide
# K, which is stricter than the parent's multiple-of-2 rule.
if c.group_size != -1:
in_features = c.partition_weight_shape[0]
if c.group_size <= 0 or in_features % c.group_size != 0:
return (
False,
f"Group size ({c.group_size}) must divide input size "
f"({in_features})",
)
if not has_zentorch_op(["zentorch_woq_repack_weight", "zentorch_woq_linear"]):
return (
False,
@@ -89,8 +108,81 @@ class ZentorchWNA16LinearKernel(CPUWNA16LinearKernel):
num_groups = weight_scale.shape[1]
return num_groups > 0 and in_features % num_groups == 0
def _maybe_process_da8w4_weights(self, layer: torch.nn.Module) -> bool:
"""Repack ``layer`` into the DA8W4 layout, or leave it untouched.
Returns False without mutating the layer when it is not eligible, so
the caller falls through to the W4A16 paths.
"""
if not envs.VLLM_CPU_INT4_W4A8:
return False
if not has_zentorch_op(["zentorch_dynamic_qlinear"]):
return False
# DA8W4 is symmetric-only, and the kernel rejects f32 activations.
if self.config.zero_points or self.config.weight_type == scalar_types.uint4:
return False
if self.config.act_type != torch.bfloat16:
return False
if not self._zentorch_woq_eligible(layer):
return False
weight_q = getattr(layer, self.w_q_name)
weight_s = getattr(layer, self.w_s_name)
weight_packed = weight_q.data if hasattr(weight_q, "data") else weight_q
weight_scale = weight_s.data if hasattr(weight_s, "data") else weight_s
bits = self.config.weight_type.mantissa
pack_factor = torch.iinfo(weight_packed.dtype).bits // bits
out_features, num_groups = weight_scale.shape[0], weight_scale.shape[1]
in_features = weight_packed.shape[1] * pack_factor
# AOCL sym_quant requires K/G to be a multiple of 4; K must be even to
# pack two s4 values per byte.
if (in_features // num_groups) % 4 != 0 or in_features % 2 != 0:
return False
if self.w_zp_name is not None:
setattr(layer, self.w_zp_name, None)
unpack_from_int32 = _import_unpack_from_int32()
weight_unpacked = unpack_from_int32(
weight_packed,
bits,
torch.Size([out_features, in_features]),
packed_dim=weight_q.packed_dim,
)
# The WOQ repack packs 8 int4 per int32, which on little-endian is the
# same byte stream as s4 [N, K/2]; the kernel takes either view.
layer._zentorch_da8w4_packed = (
torch.ops.zentorch.zentorch_woq_repack_weight.default(
weight_unpacked.to(torch.int8).contiguous()
).view(torch.int8)
)
# CT stores scales as [N, G]; DA8W4 wants per-group {G, N}.
layer._zentorch_da8w4_scale = weight_scale.t().to(torch.bfloat16).contiguous()
for param_name in (self.w_q_name, self.w_s_name):
param = getattr(layer, param_name, None)
if param is not None and hasattr(param, "data"):
param.data = torch.empty(0)
layer._zentorch_kind = "compressed_tensors_w4a8_da8w4"
layer._zentorch_da8w4 = True
layer._zentorch_processed_weights = True
logger.info_once(
"[zen_cpu] Using zentorch_dynamic_qlinear for DA8W4 (W4A8) "
"(weight_type=%s, group_size=%d)",
self.config.weight_type,
in_features // num_groups,
)
return True
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
"""Repack CT GPTQ weights into the zentorch WOQ layout.
"""Repack CT GPTQ weights into the zentorch DA8W4 or WOQ layout.
Falls back to ``CPUWNA16LinearKernel.process_weights_after_loading``
via ``super()`` when the layer doesn't satisfy
@@ -101,6 +193,9 @@ class ZentorchWNA16LinearKernel(CPUWNA16LinearKernel):
if getattr(layer, "_zentorch_processed_weights", False):
return
if self._maybe_process_da8w4_weights(layer):
return
if not self._zentorch_woq_eligible(layer):
logger.info_once(
"[zen_cpu] ZentorchWNA16 fast path not eligible for this "
@@ -186,6 +281,16 @@ class ZentorchWNA16LinearKernel(CPUWNA16LinearKernel):
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
if getattr(layer, "_zentorch_da8w4", False):
# The kernel reads bias through a raw pointer, so it must be
# contiguous; the weight and scales already are.
return torch.ops.zentorch.zentorch_dynamic_qlinear.default(
x if x.dtype == torch.bfloat16 else x.to(torch.bfloat16),
layer._zentorch_da8w4_packed,
layer._zentorch_da8w4_scale,
bias.contiguous() if bias is not None else None,
)
if getattr(layer, "_zentorch_processed_weights", False):
return torch.ops.zentorch.zentorch_woq_linear.default(
x,
@@ -51,6 +51,32 @@ from vllm.platforms import CpuArchEnum, current_platform
from vllm.utils.math_utils import round_up
logger = init_logger(__name__)
# ===========================================================================
# Weight layout
# ===========================================================================
def swigluoai_w13_interleave_perm(
experts_cls: type[mk.FusedMoEExperts],
activation: MoEActivation,
two_i: int,
device: torch.device,
) -> torch.Tensor | None:
"""Index taking a half-split w13 ``[gate | up]`` to the interleaved order
ZenDNN's ``swiglu_oai_mul`` expects, or None when no reorder is needed.
Callers apply it along whichever axis holds their output channels."""
if not getattr(experts_cls, "requires_interleaved_w13", False):
return None
if activation != MoEActivation.SWIGLUOAI:
return None
i = two_i // 2
return torch.stack(
[torch.arange(0, i, device=device), torch.arange(i, two_i, device=device)],
dim=1,
).flatten()
# ===========================================================================
# Unquantized (BF16/FP16/FP32) MoE
# ===========================================================================
@@ -0,0 +1,204 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Zentorch fused MoE experts for AMD Zen CPUs."""
from collections.abc import Callable
import torch
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm.model_executor.kernels.linear.zentorch_utils import (
_ZENTORCH_MOE_ACTIVATIONS,
_moe_activation_to_str,
has_zentorch_op,
)
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
FusedMoEParallelConfig,
FusedMoEQuantConfig,
RoutingMethodType,
)
from vllm.model_executor.layers.fused_moe.router.cpu_router import select_experts
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
kInt4Static,
kInt4Static32,
)
class ZentorchExpertsInt4(mk.FusedMoEExpertsMonolithic):
"""DA8W4 (W4A8) group-quantized monolithic MoE experts."""
# swiglu_oai_mul reads gate/up interleaved, not in the half-split order the
# weight loader leaves behind.
requires_interleaved_w13 = True
def __init__(
self,
moe_config: FusedMoEConfig,
quant_config: FusedMoEQuantConfig,
):
super().__init__(moe_config, quant_config)
# Defaults hold until process_weights_after_loading sees the layer.
self.renormalize = moe_config.routing_method in (
RoutingMethodType.Renormalize,
RoutingMethodType.RenormalizeNaive,
)
self.custom_routing_function: Callable | None = None
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
"""Capture the router config that the monolithic apply() signature
cannot carry; Custom routing has no other source for it."""
self.renormalize = layer.renormalize
self.custom_routing_function = layer.custom_routing_function
@property
def expects_unquantized_inputs(self) -> bool:
return True
@staticmethod
def activation_format() -> mk.FusedMoEActivationFormat:
return mk.FusedMoEActivationFormat.Standard
@staticmethod
def _supports_current_device() -> bool:
return has_zentorch_op(["zentorch_fused_moe", "zentorch_woq_repack_weight"])
@staticmethod
def is_supported_config(
cls: type[mk.FusedMoEExperts],
moe_config: FusedMoEConfig,
weight_key: QuantKey | None,
activation_key: QuantKey | None,
activation_format: mk.FusedMoEActivationFormat,
) -> tuple[bool, str | None]:
supported, reason = mk.FusedMoEExperts.is_supported_config(
cls, moe_config, weight_key, activation_key, activation_format
)
if not supported:
return supported, reason
if moe_config.in_dtype != torch.bfloat16:
return False, "kernel requires bfloat16 activations"
# The grouped GEMM needs tokens spread over at least two experts, which
# top_k=1 cannot satisfy once decode drops to a single token.
if moe_config.experts_per_token < 2:
return False, "kernel requires experts_per_token >= 2"
return True, None
@staticmethod
def _supports_no_act_and_mul() -> bool:
return False
@staticmethod
def _supports_activation(activation: MoEActivation) -> bool:
return _moe_activation_to_str(activation) in _ZENTORCH_MOE_ACTIVATIONS
@staticmethod
def _supports_parallel_config(
moe_parallel_config: FusedMoEParallelConfig,
) -> bool:
# apply() passes select_experts' global ids straight to the kernel, so
# every expert has to be resident on the rank.
return not moe_parallel_config.use_ep
@staticmethod
def _supports_quant_scheme(
weight_key: QuantKey | None,
activation_key: QuantKey | None,
) -> bool:
return (weight_key, activation_key) in [
(kInt4Static, None),
(kInt4Static32, None),
]
@staticmethod
def _supports_routing_method(
routing_method: RoutingMethodType,
weight_key: QuantKey | None,
activation_key: QuantKey | None,
) -> bool:
return routing_method in [
RoutingMethodType.Default,
RoutingMethodType.Renormalize,
RoutingMethodType.RenormalizeNaive,
RoutingMethodType.Custom,
]
@staticmethod
def _supports_router_logits_dtype(
router_logits_dtype: torch.dtype | None,
routing_method: RoutingMethodType,
) -> bool:
return True
def supports_expert_map(self) -> bool:
return False
def apply(
self,
hidden_states: torch.Tensor,
w1: torch.Tensor,
w2: torch.Tensor,
router_logits: torch.Tensor,
activation: MoEActivation,
global_num_experts: int,
expert_map: torch.Tensor | None,
a1q_scale: torch.Tensor | None,
apply_router_weight_on_input: bool,
# grouped topk + fused topk bias parameters
num_expert_group: int | None = None,
e_score_correction_bias: torch.Tensor | None = None,
routed_scaling_factor: float | None = None,
topk_group: int | None = None,
) -> torch.Tensor:
if apply_router_weight_on_input:
raise NotImplementedError(
"ZentorchExpertsInt4 does not support "
"apply_router_weight_on_input=True."
)
if (
self.moe_config.routing_method == RoutingMethodType.Custom
and self.custom_routing_function is None
):
raise RuntimeError(
"ZentorchExpertsInt4 needs the model's custom_routing_function "
"for RoutingMethodType.Custom."
)
topk_weights, topk_ids = select_experts(
hidden_states=hidden_states,
router_logits=router_logits,
use_grouped_topk=num_expert_group is not None,
top_k=self.moe_config.experts_per_token,
renormalize=self.renormalize,
topk_group=topk_group,
num_expert_group=num_expert_group,
custom_routing_function=self.custom_routing_function,
scoring_func="softmax",
routed_scaling_factor=(
routed_scaling_factor if routed_scaling_factor is not None else 1.0
),
e_score_correction_bias=e_score_correction_bias,
)
output = torch.empty_like(hidden_states)
torch.ops.zentorch.zentorch_fused_moe(
output,
hidden_states,
w1,
w2,
self.w1_bias,
self.w2_bias,
topk_weights,
topk_ids,
False, # skip_weighted
_moe_activation_to_str(activation),
self.w1_scale,
self.w2_scale,
)
return output
__all__ = ["ZentorchExpertsInt4"]
@@ -12,6 +12,7 @@ from compressed_tensors.quantization import (
import vllm._custom_ops as ops
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm import envs
from vllm.config.kernel import MoEBackend
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe.config import (
@@ -55,6 +56,7 @@ class WNA16MoEBackend(Enum):
BATCHED_MARLIN = "BATCHED_MARLIN"
HUMMING = "HUMMING"
CPU = "CPU"
ZEN_CPU = "ZEN_CPU"
FLASHINFER_TRTLLM = "FLASHINFER_TRTLLM"
TRITON = "TRITON"
XPU = "XPU"
@@ -98,6 +100,12 @@ def backend_to_kernel_cls(
)
return [CPUExpertsInt4]
elif backend == WNA16MoEBackend.ZEN_CPU:
from vllm.model_executor.layers.fused_moe.experts.zentorch_moe import (
ZentorchExpertsInt4,
)
return [ZentorchExpertsInt4]
elif backend == WNA16MoEBackend.EMULATION:
from vllm.model_executor.layers.fused_moe.experts.int4_emulation_moe import (
Int4EmulationTritonExperts,
@@ -117,7 +125,7 @@ def backend_to_kernel_cls(
def _get_priority_backends() -> list[WNA16MoEBackend]:
"""Get available backends in priority order based on platform and config."""
if current_platform.is_cpu():
return [WNA16MoEBackend.CPU]
return [WNA16MoEBackend.ZEN_CPU, WNA16MoEBackend.CPU]
if current_platform.is_xpu():
return [WNA16MoEBackend.XPU]
@@ -160,6 +168,27 @@ def _backend_incompatibility_reason(
from vllm.model_executor.layers.quantization.auto_gptq import AutoGPTQConfig
from vllm.model_executor.layers.quantization.moe_wna16 import MoeWNA16Config
if backend == WNA16MoEBackend.ZEN_CPU:
if not envs.VLLM_CPU_INT4_W4A8:
return "VLLM_CPU_INT4_W4A8=0 disables the DA8W4 path"
if may_have_zp:
return "zero points are not supported"
if isinstance(quant_config, MoeWNA16Config):
return "the MoeWNA16 weight layout is not supported"
if isinstance(quant_config, AutoGPTQConfig) and quant_config.desc_act:
return "GPTQ activation ordering is not supported"
if (
isinstance(quant_config, QuantizationArgs)
and quant_config.actorder == "group"
):
return "group activation ordering is not supported"
group_size = getattr(quant_config, "group_size", None)
if group_size is None or group_size <= 0:
return "DA8W4 requires group-quantized weights"
# AOCL sym_quant requires the group size to be a multiple of 4.
if group_size % 4 != 0:
return f"group size {group_size} is not a multiple of 4"
if backend == WNA16MoEBackend.TRITON:
if may_have_bias:
return "expert bias is not supported"
@@ -403,10 +432,14 @@ def make_wna16_moe_kernel(
from vllm.model_executor.layers.fused_moe.experts.xpu_moe import (
XPUExpertsWNA16,
)
from vllm.model_executor.layers.fused_moe.experts.zentorch_moe import (
ZentorchExpertsInt4,
)
# Currently, we only support TrtLlmMxint4ExpertsMonolithic, MarlinExperts,
# BatchedMarlinExperts, XPUExpertsWNA16, CPUExpertsInt4, the Humming
# grouped/indexed experts, and Int4EmulationTritonExperts
# BatchedMarlinExperts, XPUExpertsWNA16, CPUExpertsInt4,
# ZentorchExpertsInt4, the Humming grouped/indexed experts, and
# Int4EmulationTritonExperts
allowed_experts: tuple[type[mk.FusedMoEExperts], ...] = (
MarlinExperts,
BatchedMarlinExperts,
@@ -414,6 +447,7 @@ def make_wna16_moe_kernel(
TrtLlmMxint4ExpertsMonolithic,
XPUExpertsWNA16,
CPUExpertsInt4,
ZentorchExpertsInt4,
Int4EmulationTritonExperts,
Rdna3WNA16Experts,
)
@@ -1034,6 +1068,64 @@ def _process_weights_cpu(
)
def _zen_repack_s4(packed: torch.Tensor, in_features: int) -> torch.Tensor:
"""Repack CT int4 ``[E, N, K//8]`` into zentorch s4 ``[E, N, K//8]``."""
from vllm.model_executor.kernels.linear.mixed_precision.zentorch import (
_import_unpack_from_int32,
)
num_experts, out_features, _ = packed.shape
# packed_dim indexes the 2D expert slice, which is N-first, so K is dim 1.
unpacked = _import_unpack_from_int32()(
packed,
4,
torch.Size([num_experts, out_features, in_features]),
packed_dim=1,
)
# CT is already N-first, which is what the repack consumes.
repack_op = torch.ops.zentorch.zentorch_woq_repack_weight.default
return torch.stack([repack_op(w) for w in unpacked])
def _process_weights_zen_cpu(
w13: torch.Tensor,
w2: torch.Tensor,
w13_scale: torch.Tensor,
w2_scale: torch.Tensor,
w13_bias: torch.Tensor | None = None,
w2_bias: torch.Tensor | None = None,
) -> tuple[
torch.Tensor, # w13_qweight
torch.Tensor, # w2_qweight
torch.Tensor, # w13_scales
torch.Tensor, # w2_scales
torch.Tensor | None, # w13_qzeros
torch.Tensor | None, # w2_qzeros
torch.Tensor | None, # w13_input_global_scale
torch.Tensor | None, # w2_input_global_scale
torch.Tensor | None, # w13_bias
torch.Tensor | None, # w2_bias
]:
"""Zen CPU INT4 DA8W4 (W4A8) weight post-processing."""
# CT WNA16 buffers are N-first, so the scales are [E, N, G].
hidden_size = w2_scale.shape[1]
intermediate_size = w13_scale.shape[1] // 2
# The kernel reads per-group scales as [E, G, N], so N-first needs a swap.
return (
_zen_repack_s4(w13.data, hidden_size),
_zen_repack_s4(w2.data, intermediate_size),
w13_scale.data.transpose(1, 2).to(torch.bfloat16).contiguous(),
w2_scale.data.transpose(1, 2).to(torch.bfloat16).contiguous(),
None, # w13_qzeros (symmetric)
None, # w2_qzeros (symmetric)
None, # w13_input_global_scale
None, # w2_input_global_scale
w13_bias,
w2_bias,
)
def _process_weights_xpu(
layer: torch.nn.Module,
quant_config: QuantizationConfig,
@@ -1584,6 +1676,15 @@ def convert_to_wna16_moe_kernel_format(
w13_bias,
w2_bias,
)
elif backend == WNA16MoEBackend.ZEN_CPU:
return _process_weights_zen_cpu(
w13,
w2,
w13_scale,
w2_scale,
w13_bias,
w2_bias,
)
elif backend == WNA16MoEBackend.FLASHINFER_TRTLLM:
return _process_weights_flashinfer(
w13,
@@ -210,6 +210,27 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
layer.register_parameter("w2_weight_packed", w2_weight)
set_weight_attrs(w2_weight, extra_weight_attrs)
# Per-expert biases, as gpt-oss carries.
if self.moe.has_bias:
w13_num_shards = 2 if self.moe.is_act_and_mul else 1
w13_bias = torch.nn.Parameter(
torch.zeros(
num_experts,
w13_num_shards * intermediate_size_per_partition,
dtype=params_dtype,
),
requires_grad=False,
)
layer.register_parameter("w13_bias", w13_bias)
set_weight_attrs(w13_bias, extra_weight_attrs)
w2_bias = torch.nn.Parameter(
torch.zeros(num_experts, hidden_size, dtype=params_dtype),
requires_grad=False,
)
layer.register_parameter("w2_bias", w2_bias)
set_weight_attrs(w2_bias, extra_weight_attrs)
if self.strategy == "channel":
num_groups_w2 = num_groups_w13 = 1
self.group_size = -1
@@ -350,7 +371,41 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
routing_tables=layer._expert_routing_tables(),
)
if self.wna16_backend == WNA16MoEBackend.ZEN_CPU:
# Monolithic experts route inside apply(), so like the unquantized
# CPU path they need the router config that only the layer carries.
self.moe_kernel.fused_experts.process_weights_after_loading(layer)
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import (
swigluoai_w13_interleave_perm,
)
# Reorder before the conversion below, which repacks w13 for the kernel.
# The CT WNA16 buffers are N-first, so 2I is dim 1 of the packed weight
# [E, 2I, K/pack] and of its group scales [E, 2I, G], as for the bias.
perm = swigluoai_w13_interleave_perm(
self.experts_cls,
self.moe.activation,
layer.w13_weight_packed.size(1),
layer.w13_weight_packed.device,
)
if perm is not None:
replace_parameter(
layer,
"w13_weight_packed",
layer.w13_weight_packed.data[:, perm].contiguous(),
)
replace_parameter(
layer,
"w13_weight_scale",
layer.w13_weight_scale.data[:, perm].contiguous(),
)
if self.moe.has_bias:
replace_parameter(
layer, "w13_bias", layer.w13_bias.data[:, perm].contiguous()
)
# Process weights using the shared oracle infrastructure
converted = convert_to_wna16_moe_kernel_format(
backend=self.wna16_backend,
@@ -363,6 +418,8 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
w2_scale=layer.w2_weight_scale,
w13_qzeros=getattr(layer, "w13_weight_zero_point", None),
w2_qzeros=getattr(layer, "w2_weight_zero_point", None),
w13_bias=getattr(layer, "w13_bias", None),
w2_bias=getattr(layer, "w2_bias", None),
)
if converted is None:
@@ -378,8 +435,8 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
w2_qzeros,
w13_input_global_scale,
w2_input_global_scale,
_, # w13_bias
_, # w2_bias
w13_bias,
w2_bias,
) = converted
# Replace common parameters
@@ -388,6 +445,11 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
replace_parameter(layer, "w13_weight_scale", w13_scales)
replace_parameter(layer, "w2_weight_scale", w2_scales)
if w13_bias is not None:
replace_parameter(layer, "w13_bias", w13_bias)
if w2_bias is not None:
replace_parameter(layer, "w2_bias", w2_bias)
# CPU fused_experts_cpu requires zero points even for symmetric quant.
# EMULATION bakes ZP into the dequantized bf16 weights — ZP is None.
if (
@@ -437,6 +499,8 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
num_bits=self.num_bits,
w1_zp=getattr(layer, "w13_weight_zero_point", None),
w2_zp=getattr(layer, "w2_weight_zero_point", None),
w1_bias=getattr(layer, "w13_bias", None),
w2_bias=getattr(layer, "w2_bias", None),
gemm1_clamp_limit=getattr(layer, "swiglu_limit", None),
gemm1_alpha=getattr(layer, "swiglu_alpha", None),
gemm1_beta=getattr(layer, "swiglu_beta", None),
+4 -1
View File
@@ -76,6 +76,8 @@ _GPT_OSS_STREAMED_EXPERT_SUFFIX_TO_SHARD = {
"w2_bias": "gpt_oss_w2",
"w13_weight_scale": "gpt_oss_w13",
"w2_weight_scale": "gpt_oss_w2",
"w13_weight_packed": "gpt_oss_w13",
"w2_weight_packed": "gpt_oss_w2",
}
@@ -263,7 +265,7 @@ class GptOssRoutedExperts(RoutedExperts):
) -> None:
tp_rank = self.moe_config.moe_parallel_config.tp_rank
is_w13 = shard_id == "gpt_oss_w13"
is_weight = weight_name.endswith("_weight")
is_weight = weight_name.endswith(("_weight", "_weight_packed"))
is_partitioned_scale = weight_name.endswith("_weight_scale")
if is_weight:
@@ -347,6 +349,7 @@ class GptOssRoutedExperts(RoutedExperts):
) or quant_method_name in (
"CompressedTensorsW4A4Nvfp4MoEMethod",
"Nvfp4OnlineMoEMethod",
"CompressedTensorsWNA16MoEMethod",
):
self._load_packed_expert(expert_data, loaded_weight, weight_name, shard_id)
else: