diff --git a/.buildkite/hardware_tests/cpu.yaml b/.buildkite/hardware_tests/cpu.yaml index b50a082727..d54197f836 100644 --- a/.buildkite/hardware_tests/cpu.yaml +++ b/.buildkite/hardware_tests/cpu.yaml @@ -24,6 +24,9 @@ steps: - tests/kernels/test_onednn.py - tests/kernels/test_awq_int4_to_int8.py - tests/kernels/quantization/test_cpu_fp8_scaled_mm.py + - tests/kernels/quantization/test_zen_da8w4.py + - vllm/model_executor/kernels/linear/mixed_precision/zentorch.py + - vllm/model_executor/layers/fused_moe/experts/zentorch_moe.py - tests/kernels/mamba/cpu/test_cpu_gdn_ops.py - tests/kernels/mamba/test_cpu_short_conv.py - tests/kernels/mamba/test_causal_conv1d.py @@ -46,6 +49,7 @@ steps: tests/kernels/test_onednn.py \ tests/kernels/test_awq_int4_to_int8.py \ tests/kernels/quantization/test_cpu_fp8_scaled_mm.py \ + tests/kernels/quantization/test_zen_da8w4.py \ tests/kernels/mamba/cpu/test_cpu_gdn_ops.py \ tests/kernels/mamba/test_causal_conv1d.py \ tests/kernels/mamba/test_mamba_ssm.py \ diff --git a/tests/kernels/quantization/test_zen_da8w4.py b/tests/kernels/quantization/test_zen_da8w4.py new file mode 100644 index 0000000000..924b7a865f --- /dev/null +++ b/tests/kernels/quantization/test_zen_da8w4.py @@ -0,0 +1,661 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Tests for the zentorch DA8W4 (W4A8) int4 linear and MoE paths on Zen CPUs. + +The zentorch ops are mocked with reference implementations when zentorch is not +installed, so the layout and dispatch contracts are covered in CI. +""" + +import dataclasses + +import pytest +import torch +from compressed_tensors.compressors.pack_quantized.helpers import pack_to_int32 + +from tests.kernels.quant_utils import ref_dynamic_per_token_quant +from vllm.model_executor.kernels.linear.mixed_precision.MPLinearKernel import ( + MPLinearLayerConfig, +) +from vllm.model_executor.kernels.linear.mixed_precision.zentorch import ( + ZentorchWNA16LinearKernel, + _import_unpack_from_int32, +) +from vllm.model_executor.layers.fused_moe.oracle import int_wna16 +from vllm.model_executor.layers.fused_moe.oracle.int_wna16 import WNA16MoEBackend +from vllm.platforms import current_platform +from vllm.scalar_type import scalar_types + +if not current_platform.is_cpu(): + pytest.skip("skipping CPU-only tests", allow_module_level=True) + +GROUP_SIZE = 128 +IN_FEATURES = 512 +OUT_FEATURES = 128 + + +def _unpack_s4(packed: torch.Tensor, in_features: int) -> torch.Tensor: + """int8 [N, K/2] (or int32 [N, K/8]) packed s4 -> float32 [N, K].""" + words = packed.view(torch.int32) + out = torch.zeros(words.shape[0], in_features, dtype=torch.float32) + for i in range(8): + nibble = (words >> (4 * i)) & 0xF + out[:, i::8] = torch.where(nibble > 7, nibble - 16, nibble).float() + return out + + +def _repack_s4(unpacked: torch.Tensor) -> torch.Tensor: + """Reference zentorch_woq_repack_weight: int8 [N, K] -> int32 [N, K/8].""" + n, k = unpacked.shape + values = unpacked.to(torch.int32).reshape(n, k // 8, 8) + out = torch.zeros(n, k // 8, dtype=torch.int32) + for i in range(8): + out |= (values[:, :, i] & 0xF) << (4 * i) + return out + + +def _quantize_per_group( + weight: torch.Tensor, group_size: int +) -> tuple[torch.Tensor, torch.Tensor]: + """Symmetric int4 per-group quantization -> (int8 in [-8, 7], scale [N, G]).""" + out_features, in_features = weight.shape + grouped = weight.float().reshape( + out_features, in_features // group_size, group_size + ) + scale = grouped.abs().amax(dim=-1) / 8.0 + quantized = (grouped / scale.unsqueeze(-1)).round().clamp(-8, 7).to(torch.int8) + return quantized.reshape(out_features, in_features), scale.to(torch.bfloat16) + + +def _dequantize_per_group( + quantized: torch.Tensor, scale: torch.Tensor, group_size: int +) -> torch.Tensor: + """Dequantize with [N, G] scales into a float32 [N, K] weight.""" + return quantized.float() * scale.float().repeat_interleave(group_size, dim=1) + + +def _dynamic_quant_matmul( + x: torch.Tensor, weight: torch.Tensor, bias: torch.Tensor | None +) -> torch.Tensor: + """DA8W4 reference: per-token int8 activation quant against a bf16 weight.""" + x_q, scale = ref_dynamic_per_token_quant(x.float(), torch.int8) + out = (x_q.float() @ weight.t()) * scale + if bias is not None: + out = out + bias.float() + return out + + +@pytest.fixture +def mock_zentorch_ops(): + """Register reference zentorch ops when zentorch is not installed.""" + if hasattr(torch.ops.zentorch, "zentorch_dynamic_qlinear"): + yield None + return + + calls: dict[str, tuple] = {} + + def _dynamic_qlinear(input, weight, weight_scales, bias=None, zentorch_op_name=""): + calls["dynamic_qlinear"] = (input, weight, weight_scales, bias) + in_features = weight.view(torch.int32).shape[1] * 8 + group_size = in_features // weight_scales.shape[0] + # weight_scales is [G, N]; expand to a [N, K] dequantized weight. + scales = weight_scales.float().t().repeat_interleave(group_size, dim=1) + weight_deq = _unpack_s4(weight, in_features) * scales + out = _dynamic_quant_matmul(input, weight_deq, bias) + return out.to(input.dtype) + + def _fused_moe( + output, + input, + w13, + w2, + w13_bias, + w2_bias, + topk_weights, + topk_id, + skip_weighted, + act, + w13_scales=None, + w2_scales=None, + zentorch_op_name="", + ): + calls["fused_moe"] = ( + input, + w13, + w2, + topk_weights, + topk_id, + skip_weighted, + act, + w13_scales, + w2_scales, + ) + output.zero_() + + lib_def = torch.library.Library("zentorch", "DEF") + lib_def.define("zentorch_woq_repack_weight(Tensor unpacked_weight) -> Tensor") + lib_def.define( + "zentorch_dynamic_qlinear(Tensor input, Tensor weight, " + "Tensor weight_scales, Tensor? bias=None, *, " + "str zentorch_op_name='zentorch::zentorch_dynamic_qlinear') -> Tensor" + ) + lib_def.define( + "zentorch_fused_moe(Tensor(a!) output, Tensor input, Tensor w13, " + "Tensor w2, Tensor? w13_bias, Tensor? w2_bias, Tensor topk_weights, " + "Tensor topk_id, bool skip_weighted, str act, " + "Tensor? w13_scales=None, Tensor? w2_scales=None, *, " + "str zentorch_op_name='zentorch::zentorch_fused_moe') -> ()" + ) + + lib_impl = torch.library.Library("zentorch", "IMPL", "CPU") + lib_impl.impl("zentorch_woq_repack_weight", _repack_s4) + lib_impl.impl("zentorch_dynamic_qlinear", _dynamic_qlinear) + lib_impl.impl("zentorch_fused_moe", _fused_moe) + + yield calls + + lib_impl._destroy() + lib_def._destroy() + + +def _make_config( + act_type: torch.dtype = torch.bfloat16, + weight_type=scalar_types.uint4b8, + zero_points: bool = False, + group_size: int = GROUP_SIZE, +) -> MPLinearLayerConfig: + return MPLinearLayerConfig( + full_weight_shape=(IN_FEATURES, OUT_FEATURES), + partition_weight_shape=(IN_FEATURES, OUT_FEATURES), + weight_type=weight_type, + act_type=act_type, + group_size=group_size, + zero_points=zero_points, + ) + + +def _make_layer(group_size: int = GROUP_SIZE) -> tuple[torch.nn.Module, torch.Tensor]: + """Build a layer holding a CT-packed int4 weight, plus its dequantized form.""" + torch.manual_seed(0) + weight = torch.randn(OUT_FEATURES, IN_FEATURES, dtype=torch.bfloat16) + quantized, scale = _quantize_per_group(weight, group_size) + + # compressed-tensors stores [N, K//8] int32, packed along the input dim. + packed = pack_to_int32(quantized, 4, packed_dim=1) + + layer = torch.nn.Module() + layer.weight_packed = torch.nn.Parameter(packed, requires_grad=False) + layer.weight_packed.input_dim = 1 + layer.weight_packed.packed_dim = 1 + layer.weight_scale = torch.nn.Parameter(scale, requires_grad=False) + layer.weight_zero_point = None + return layer, _dequantize_per_group(quantized, scale, group_size) + + +def _make_kernel(config: MPLinearLayerConfig) -> ZentorchWNA16LinearKernel: + return ZentorchWNA16LinearKernel( + config, "weight_packed", "weight_scale", "weight_zero_point" + ) + + +# --------------------------------------------------------------------------- +# Dense DA8W4 +# --------------------------------------------------------------------------- + + +def test_da8w4_eligible_for_symmetric_bf16_layer(mock_zentorch_ops, monkeypatch): + monkeypatch.setattr( + "vllm.model_executor.kernels.linear.mixed_precision.zentorch." + "current_platform.is_zen_cpu", + lambda: True, + ) + layer, _ = _make_layer() + kernel = _make_kernel(_make_config()) + assert kernel._maybe_process_da8w4_weights(layer) + + +@pytest.mark.parametrize( + "config_kwargs,reason", + [ + ({"act_type": torch.float32}, "f32 activations are rejected"), + ( + {"weight_type": scalar_types.uint4, "zero_points": True}, + "asymmetric int4 is unsupported", + ), + ], +) +def test_da8w4_not_eligible(mock_zentorch_ops, monkeypatch, config_kwargs, reason): + monkeypatch.setattr( + "vllm.model_executor.kernels.linear.mixed_precision.zentorch." + "current_platform.is_zen_cpu", + lambda: True, + ) + layer, _ = _make_layer() + kernel = _make_kernel(_make_config(**config_kwargs)) + assert not kernel._maybe_process_da8w4_weights(layer), reason + + +def test_da8w4_not_eligible_when_env_disabled(mock_zentorch_ops, monkeypatch): + monkeypatch.setattr( + "vllm.model_executor.kernels.linear.mixed_precision.zentorch." + "current_platform.is_zen_cpu", + lambda: True, + ) + monkeypatch.setattr( + "vllm.model_executor.kernels.linear.mixed_precision.zentorch." + "envs.VLLM_CPU_INT4_W4A8", + False, + ) + layer, _ = _make_layer() + kernel = _make_kernel(_make_config()) + assert not kernel._maybe_process_da8w4_weights(layer) + + +def test_da8w4_process_weights_layout(mock_zentorch_ops, monkeypatch): + monkeypatch.setattr( + "vllm.model_executor.kernels.linear.mixed_precision.zentorch." + "current_platform.is_zen_cpu", + lambda: True, + ) + layer, _ = _make_layer() + kernel = _make_kernel(_make_config()) + kernel.process_weights_after_loading(layer) + + assert layer._zentorch_da8w4 + assert layer._zentorch_kind == "compressed_tensors_w4a8_da8w4" + # Packed s4 holds two nibbles per byte, and scales transpose to {G, N}. + assert layer._zentorch_da8w4_packed.dtype == torch.int8 + assert layer._zentorch_da8w4_packed.shape == (OUT_FEATURES, IN_FEATURES // 2) + assert layer._zentorch_da8w4_scale.dtype == torch.bfloat16 + assert layer._zentorch_da8w4_scale.shape == ( + IN_FEATURES // GROUP_SIZE, + OUT_FEATURES, + ) + assert layer._zentorch_da8w4_packed.is_contiguous() + assert layer._zentorch_da8w4_scale.is_contiguous() + # The checkpoint parameters are released once repacked. + assert layer.weight_packed.numel() == 0 + assert layer.weight_scale.numel() == 0 + + +def test_da8w4_apply_weights_matches_dequantized_reference( + mock_zentorch_ops, monkeypatch +): + monkeypatch.setattr( + "vllm.model_executor.kernels.linear.mixed_precision.zentorch." + "current_platform.is_zen_cpu", + lambda: True, + ) + layer, weight_deq = _make_layer() + kernel = _make_kernel(_make_config()) + kernel.process_weights_after_loading(layer) + + x = torch.randn(4, IN_FEATURES, dtype=torch.bfloat16) + bias = torch.randn(OUT_FEATURES, dtype=torch.bfloat16) + out = kernel.apply_weights(layer, x, bias) + + expected = _dynamic_quant_matmul(x, weight_deq, bias) + assert out.dtype == torch.bfloat16 + torch.testing.assert_close(out.float(), expected, rtol=2e-2, atol=2e-2) + + +def test_da8w4_falls_back_to_w4a16_when_op_missing(monkeypatch): + """Without zentorch_dynamic_qlinear the layer must not take the DA8W4 path.""" + monkeypatch.setattr( + "vllm.model_executor.kernels.linear.mixed_precision.zentorch.has_zentorch_op", + lambda ops: "zentorch_dynamic_qlinear" not in ops, + ) + layer, _ = _make_layer() + kernel = _make_kernel(_make_config()) + assert not kernel._maybe_process_da8w4_weights(layer) + + +def test_can_implement_requires_zen_cpu(monkeypatch): + monkeypatch.setattr( + "vllm.model_executor.kernels.linear.mixed_precision.zentorch." + "current_platform.is_zen_cpu", + lambda: False, + ) + ok, reason = ZentorchWNA16LinearKernel.can_implement(_make_config()) + assert not ok + assert reason is not None + + +def test_can_implement_accepts_shapes_cpuwna16_rejects(monkeypatch): + """CPUWNA16's N/K % 32 rule is a oneDNN packing constraint zentorch lacks.""" + from vllm.model_executor.kernels.linear.mixed_precision.cpu import ( + CPUWNA16LinearKernel, + ) + + monkeypatch.setattr( + "vllm.model_executor.kernels.linear.mixed_precision.zentorch." + "current_platform.is_zen_cpu", + lambda: True, + ) + monkeypatch.setattr( + "vllm.model_executor.kernels.linear.mixed_precision.cpu." + "current_platform.is_cpu", + lambda: True, + ) + monkeypatch.setattr( + "vllm.model_executor.kernels.linear.mixed_precision.zentorch.has_zentorch_op", + lambda ops: True, + ) + + unaligned = dataclasses.replace( + _make_config(), partition_weight_shape=(IN_FEATURES, OUT_FEATURES - 16) + ) + assert unaligned.partition_weight_shape[1] % 32 != 0 + + ok, _ = CPUWNA16LinearKernel.can_implement(unaligned) + assert not ok, "the parent is expected to turn this shape away" + + ok, reason = ZentorchWNA16LinearKernel.can_implement(unaligned) + assert ok, reason + + +@pytest.mark.parametrize( + "group_size,expected_ok", + [ + (-1, True), # per-channel, one group spanning K + (GROUP_SIZE, True), + (IN_FEATURES, True), + (6, False), # even, so the parent's multiple-of-2 rule would allow it + (0, False), + ], +) +def test_can_implement_group_size_must_divide_k(monkeypatch, group_size, expected_ok): + """ZenDNN WOQ groups along K, so a group that straddles the end is unusable.""" + monkeypatch.setattr( + "vllm.model_executor.kernels.linear.mixed_precision.zentorch." + "current_platform.is_zen_cpu", + lambda: True, + ) + monkeypatch.setattr( + "vllm.model_executor.kernels.linear.mixed_precision.zentorch.has_zentorch_op", + lambda ops: True, + ) + + ok, reason = ZentorchWNA16LinearKernel.can_implement( + _make_config(group_size=group_size) + ) + assert ok is expected_ok, reason + if not expected_ok: + assert reason is not None + assert "must divide input size" in reason + + +# --------------------------------------------------------------------------- +# MoE DA8W4 +# --------------------------------------------------------------------------- + +NUM_EXPERTS = 4 +HIDDEN = 256 +INTERMEDIATE = 128 + + +def _make_moe_weights(group_size: int = 32): + """Build CT-layout MoE weights: w13 [E, 2I, H//8], w2 [E, H, I//8].""" + torch.manual_seed(0) + w13_q, w13_s, w2_q, w2_s = [], [], [], [] + for _ in range(NUM_EXPERTS): + w13 = torch.randn(2 * INTERMEDIATE, HIDDEN, dtype=torch.bfloat16) + q13, s13 = _quantize_per_group(w13, group_size) + # CT is N-first and packs along the input dim: [N, K//8] per expert. + w13_q.append(pack_to_int32(q13, 4, packed_dim=1)) + w13_s.append(s13) + + w2 = torch.randn(HIDDEN, INTERMEDIATE, dtype=torch.bfloat16) + q2, s2 = _quantize_per_group(w2, group_size) + w2_q.append(pack_to_int32(q2, 4, packed_dim=1)) + w2_s.append(s2) + return ( + torch.stack(w13_q), + torch.stack(w2_q), + torch.stack(w13_s), + torch.stack(w2_s), + ) + + +def test_zen_cpu_first_in_cpu_backend_priority(monkeypatch): + monkeypatch.setattr(int_wna16.current_platform, "is_cpu", lambda: True) + backends = int_wna16._get_priority_backends() + assert backends[0] == WNA16MoEBackend.ZEN_CPU + # The generic CPU backend stays available as a fallback. + assert WNA16MoEBackend.CPU in backends + + +def test_zen_cpu_process_weights_layout(mock_zentorch_ops): + group_size = 32 + w13, w2, w13_scale, w2_scale = _make_moe_weights(group_size) + assert w13.shape == (NUM_EXPERTS, 2 * INTERMEDIATE, HIDDEN // 8) + assert w2.shape == (NUM_EXPERTS, HIDDEN, INTERMEDIATE // 8) + + converted = int_wna16._process_weights_zen_cpu(w13, w2, w13_scale, w2_scale) + w13_out, w2_out, w13_s_out, w2_s_out = converted[:4] + + # zentorch consumes [E, N, K/8] packed s4 with per-group [E, G, N] scales. + assert w13_out.shape == (NUM_EXPERTS, 2 * INTERMEDIATE, HIDDEN // 8) + assert w2_out.shape == (NUM_EXPERTS, HIDDEN, INTERMEDIATE // 8) + assert w13_out.dtype == w2_out.dtype == torch.int32 + assert w13_s_out.shape == (NUM_EXPERTS, HIDDEN // group_size, 2 * INTERMEDIATE) + assert w2_s_out.shape == (NUM_EXPERTS, INTERMEDIATE // group_size, HIDDEN) + assert w13_s_out.dtype == w2_s_out.dtype == torch.bfloat16 + # Symmetric checkpoints carry no zero points into the kernel. + assert converted[8] is None and converted[9] is None + + +def test_zen_cpu_repack_is_value_exact(mock_zentorch_ops): + """The repacked weight must dequantize back to the checkpoint values.""" + group_size = 32 + w13, w2, w13_scale, w2_scale = _make_moe_weights(group_size) + w13_out = int_wna16._process_weights_zen_cpu(w13, w2, w13_scale, w2_scale)[0] + + expected = _import_unpack_from_int32()( + w13, + 4, + torch.Size([NUM_EXPERTS, 2 * INTERMEDIATE, HIDDEN]), + packed_dim=1, + ) + for expert in range(NUM_EXPERTS): + torch.testing.assert_close( + _unpack_s4(w13_out[expert], HIDDEN), + expected[expert].float(), + ) + + +@pytest.mark.parametrize( + "group_size,expected_ok", + [(32, True), (128, True), (-1, False), (2, False)], +) +def test_zen_cpu_group_size_gating(group_size, expected_ok): + quant_config = type("Args", (), {"group_size": group_size})() + reason = int_wna16._backend_incompatibility_reason( + WNA16MoEBackend.ZEN_CPU, + moe_config=None, + quant_config=quant_config, + may_have_zp=False, + may_have_bias=False, + allow_tile_padding=True, + ) + assert (reason is None) == expected_ok + + +@pytest.mark.parametrize( + "may_have_zp,may_have_bias,expected_ok", + [(True, False, False), (False, True, True), (True, True, False)], +) +def test_zen_cpu_rejects_zero_points_but_takes_bias( + may_have_zp, may_have_bias, expected_ok +): + quant_config = type("Args", (), {"group_size": 128})() + reason = int_wna16._backend_incompatibility_reason( + WNA16MoEBackend.ZEN_CPU, + moe_config=None, + quant_config=quant_config, + may_have_zp=may_have_zp, + may_have_bias=may_have_bias, + allow_tile_padding=True, + ) + assert (reason is None) == expected_ok + + +def test_zen_cpu_rejects_moe_wna16_layout(): + from vllm.model_executor.layers.quantization.moe_wna16 import MoeWNA16Config + + quant_config = MoeWNA16Config( + linear_quant_method="gptq", + weight_bits=4, + group_size=128, + has_zp=False, + lm_head_quantized=False, + modules_to_not_convert=None, + full_config={}, + ) + reason = int_wna16._backend_incompatibility_reason( + WNA16MoEBackend.ZEN_CPU, + moe_config=None, + quant_config=quant_config, + may_have_zp=False, + may_have_bias=False, + allow_tile_padding=True, + ) + assert reason is not None + + +def test_zen_cpu_disabled_by_env(monkeypatch): + monkeypatch.setattr(int_wna16.envs, "VLLM_CPU_INT4_W4A8", False) + quant_config = type("Args", (), {"group_size": 128})() + reason = int_wna16._backend_incompatibility_reason( + WNA16MoEBackend.ZEN_CPU, + moe_config=None, + quant_config=quant_config, + may_have_zp=False, + may_have_bias=False, + allow_tile_padding=True, + ) + assert reason is not None + + +def test_zen_cpu_backend_maps_to_experts_class(): + from vllm.model_executor.layers.fused_moe.experts.zentorch_moe import ( + ZentorchExpertsInt4, + ) + + assert int_wna16.backend_to_kernel_cls(WNA16MoEBackend.ZEN_CPU) == [ + ZentorchExpertsInt4 + ] + + +def test_zen_experts_support_predicates(): + from vllm.model_executor.kernels.linear.zentorch_utils import ( + _ZENTORCH_MOE_ACTIVATIONS, + ) + from vllm.model_executor.layers.fused_moe.activation import MoEActivation + from vllm.model_executor.layers.fused_moe.experts.zentorch_moe import ( + ZentorchExpertsInt4, + ) + from vllm.model_executor.layers.quantization.utils.quant_utils import ( + kInt4Static, + kInt4Static32, + kInt8StaticChannelSym, + ) + + assert ZentorchExpertsInt4._supports_quant_scheme(kInt4Static, None) + assert ZentorchExpertsInt4._supports_quant_scheme(kInt4Static32, None) + assert not ZentorchExpertsInt4._supports_quant_scheme(kInt8StaticChannelSym, None) + for act in MoEActivation: + expected = act.value in _ZENTORCH_MOE_ACTIVATIONS + assert ZentorchExpertsInt4._supports_activation(act) == expected + assert ZentorchExpertsInt4.requires_interleaved_w13 + assert not ZentorchExpertsInt4._supports_no_act_and_mul() + assert ZentorchExpertsInt4._supports_parallel_config( + type("Par", (), {"use_ep": False})() + ) + assert not ZentorchExpertsInt4._supports_parallel_config( + type("Par", (), {"use_ep": True})() + ) + + +def test_zen_experts_take_custom_routing(): + """Models with their own router (gemma-4) reach select_experts through the + config captured off the layer, since apply() cannot be handed the callable.""" + from types import SimpleNamespace + + from tests.kernels.moe.test_cpu_fused_moe import _StubMoELayer + from vllm.model_executor.layers.fused_moe.activation import MoEActivation + from vllm.model_executor.layers.fused_moe.config import RoutingMethodType + from vllm.model_executor.layers.fused_moe.experts.zentorch_moe import ( + ZentorchExpertsInt4, + ) + + assert ZentorchExpertsInt4._supports_routing_method( + RoutingMethodType.Custom, None, None + ) + + def routing_fn(**kwargs): + raise AssertionError("not called") + + layer = _StubMoELayer( + torch.zeros(NUM_EXPERTS, 2 * INTERMEDIATE, HIDDEN), + torch.zeros(NUM_EXPERTS, HIDDEN, INTERMEDIATE), + MoEActivation.SILU, + ) + layer.renormalize = True + layer.custom_routing_function = routing_fn + + experts = SimpleNamespace(renormalize=False, custom_routing_function=None) + ZentorchExpertsInt4.process_weights_after_loading(experts, layer) + + assert experts.custom_routing_function is routing_fn + assert experts.renormalize is True + + +def _swigluoai_perm(activation, experts_cls=None): + from vllm.model_executor.layers.fused_moe.experts.cpu_moe import ( + swigluoai_w13_interleave_perm, + ) + from vllm.model_executor.layers.fused_moe.experts.zentorch_moe import ( + ZentorchExpertsInt4, + ) + + return swigluoai_w13_interleave_perm( + experts_cls or ZentorchExpertsInt4, + activation, + 2 * INTERMEDIATE, + torch.device("cpu"), + ) + + +def test_swigluoai_perm_only_for_zen_swigluoai(): + from vllm.model_executor.layers.fused_moe.activation import MoEActivation + + assert _swigluoai_perm(MoEActivation.SWIGLUOAI) is not None + # Other activations keep the half-split layout the loader leaves. + assert _swigluoai_perm(MoEActivation.SILU) is None + # object stands in for any kernel that never set requires_interleaved_w13. + assert _swigluoai_perm(MoEActivation.SWIGLUOAI, experts_cls=object) is None + + +def test_swigluoai_perm_interleaves_weights_scales_and_bias(mock_zentorch_ops): + """The permutation is a gather on w13's output-channel axis, which is dim 1 + for the N-first packed weights and their group scales.""" + from vllm.model_executor.layers.fused_moe.activation import MoEActivation + + group_size = 32 + w13, w2, w13_scale, w2_scale = _make_moe_weights(group_size) + bias = torch.randn(NUM_EXPERTS, 2 * INTERMEDIATE, dtype=torch.bfloat16) + perm = _swigluoai_perm(MoEActivation.SWIGLUOAI) + + baseline = int_wna16._process_weights_zen_cpu(w13, w2, w13_scale, w2_scale)[0] + permuted = int_wna16._process_weights_zen_cpu( + w13[:, perm].contiguous(), w2, w13_scale[:, perm].contiguous(), w2_scale + )[0] + + for expert in range(NUM_EXPERTS): + before = _unpack_s4(baseline[expert], HIDDEN) + after = _unpack_s4(permuted[expert], HIDDEN) + torch.testing.assert_close(after[0::2], before[:INTERMEDIATE]) + torch.testing.assert_close(after[1::2], before[INTERMEDIATE:]) + + torch.testing.assert_close(bias[:, perm][:, 0::2], bias[:, :INTERMEDIATE]) + torch.testing.assert_close(bias[:, perm][:, 1::2], bias[:, INTERMEDIATE:]) diff --git a/vllm/envs.py b/vllm/envs.py index 5daa4fd3e6..e63ed5cc02 100755 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -912,7 +912,8 @@ environment_variables: dict[str, Callable[[], Any]] = { "VLLM_ZENTORCH_WEIGHT_PREPACK": lambda: bool( int(os.getenv("VLLM_ZENTORCH_WEIGHT_PREPACK", "1")) ), - # (CPU backend only) whether to use SGLang INT4 W4A8 kernels for AWQ. + # (CPU backend only) whether to use SGLang INT4 W4A8 kernels for AWQ, and + # on Zen CPUs whether to serve int4 checkpoints as DA8W4 rather than W4A16. "VLLM_CPU_INT4_W4A8": lambda: bool(int(os.getenv("VLLM_CPU_INT4_W4A8", "1"))), # If the env var is set, Ray Compiled Graph uses the specified # channel type to communicate between workers belonging to diff --git a/vllm/model_executor/kernels/linear/mixed_precision/zentorch.py b/vllm/model_executor/kernels/linear/mixed_precision/zentorch.py index 90968ab2c2..c62a705076 100644 --- a/vllm/model_executor/kernels/linear/mixed_precision/zentorch.py +++ b/vllm/model_executor/kernels/linear/mixed_precision/zentorch.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Zentorch W4A16 GPTQ weight-only-quantized linear kernel for AMD Zen CPUs. +"""Zentorch int4 weight-quantized linear kernels for AMD Zen CPUs. Selected by ``choose_mp_linear_kernel`` ahead of the generic oneDNN-backed ``CPUWNA16LinearKernel``. When ``can_implement`` rejects a layer, the selector @@ -9,12 +9,13 @@ falls through to the next kernel in ``_POSSIBLE_KERNELS[PlatformEnum.CPU]``. import torch +from vllm import envs from vllm.logger import init_logger from vllm.model_executor.kernels.linear.zentorch_utils import has_zentorch_op from vllm.platforms import current_platform from vllm.scalar_type import scalar_types -from .cpu import CPUWNA16LinearKernel +from .cpu import _CPUWNA16_SUPPORTED_QUANT_TYPES, CPUWNA16LinearKernel from .MPLinearKernel import MPLinearLayerConfig logger = init_logger(__name__) @@ -34,17 +35,35 @@ def _import_unpack_from_int32(): class ZentorchWNA16LinearKernel(CPUWNA16LinearKernel): - """W4A16 GPTQ kernel backed by ``torch.ops.zentorch.zentorch_woq_linear``.""" + """Int4 kernel backed by ``zentorch_dynamic_qlinear`` (DA8W4) or + ``zentorch_woq_linear`` (W4A16).""" @classmethod def can_implement(cls, c: MPLinearLayerConfig) -> tuple[bool, str | None]: - ok, reason = super().can_implement(c) - if not ok: - return ok, reason - if not current_platform.is_zen_cpu(): return False, "ZentorchWNA16 requires an AMD Zen CPU." + # Not super(): its N/K multiple-of-32 rule is a oneDNN packing + # constraint the zentorch repack does not share. + if c.weight_type not in _CPUWNA16_SUPPORTED_QUANT_TYPES: + return ( + False, + f"Quant type ({c.weight_type}) not supported by " + "CPUWNA16, supported types are: " + f"{_CPUWNA16_SUPPORTED_QUANT_TYPES}", + ) + + # -1 is per-channel, one group spanning K. Anything else has to divide + # K, which is stricter than the parent's multiple-of-2 rule. + if c.group_size != -1: + in_features = c.partition_weight_shape[0] + if c.group_size <= 0 or in_features % c.group_size != 0: + return ( + False, + f"Group size ({c.group_size}) must divide input size " + f"({in_features})", + ) + if not has_zentorch_op(["zentorch_woq_repack_weight", "zentorch_woq_linear"]): return ( False, @@ -89,8 +108,81 @@ class ZentorchWNA16LinearKernel(CPUWNA16LinearKernel): num_groups = weight_scale.shape[1] return num_groups > 0 and in_features % num_groups == 0 + def _maybe_process_da8w4_weights(self, layer: torch.nn.Module) -> bool: + """Repack ``layer`` into the DA8W4 layout, or leave it untouched. + + Returns False without mutating the layer when it is not eligible, so + the caller falls through to the W4A16 paths. + """ + if not envs.VLLM_CPU_INT4_W4A8: + return False + + if not has_zentorch_op(["zentorch_dynamic_qlinear"]): + return False + + # DA8W4 is symmetric-only, and the kernel rejects f32 activations. + if self.config.zero_points or self.config.weight_type == scalar_types.uint4: + return False + if self.config.act_type != torch.bfloat16: + return False + + if not self._zentorch_woq_eligible(layer): + return False + + weight_q = getattr(layer, self.w_q_name) + weight_s = getattr(layer, self.w_s_name) + weight_packed = weight_q.data if hasattr(weight_q, "data") else weight_q + weight_scale = weight_s.data if hasattr(weight_s, "data") else weight_s + + bits = self.config.weight_type.mantissa + pack_factor = torch.iinfo(weight_packed.dtype).bits // bits + out_features, num_groups = weight_scale.shape[0], weight_scale.shape[1] + in_features = weight_packed.shape[1] * pack_factor + + # AOCL sym_quant requires K/G to be a multiple of 4; K must be even to + # pack two s4 values per byte. + if (in_features // num_groups) % 4 != 0 or in_features % 2 != 0: + return False + + if self.w_zp_name is not None: + setattr(layer, self.w_zp_name, None) + + unpack_from_int32 = _import_unpack_from_int32() + + weight_unpacked = unpack_from_int32( + weight_packed, + bits, + torch.Size([out_features, in_features]), + packed_dim=weight_q.packed_dim, + ) + # The WOQ repack packs 8 int4 per int32, which on little-endian is the + # same byte stream as s4 [N, K/2]; the kernel takes either view. + layer._zentorch_da8w4_packed = ( + torch.ops.zentorch.zentorch_woq_repack_weight.default( + weight_unpacked.to(torch.int8).contiguous() + ).view(torch.int8) + ) + # CT stores scales as [N, G]; DA8W4 wants per-group {G, N}. + layer._zentorch_da8w4_scale = weight_scale.t().to(torch.bfloat16).contiguous() + + for param_name in (self.w_q_name, self.w_s_name): + param = getattr(layer, param_name, None) + if param is not None and hasattr(param, "data"): + param.data = torch.empty(0) + + layer._zentorch_kind = "compressed_tensors_w4a8_da8w4" + layer._zentorch_da8w4 = True + layer._zentorch_processed_weights = True + logger.info_once( + "[zen_cpu] Using zentorch_dynamic_qlinear for DA8W4 (W4A8) " + "(weight_type=%s, group_size=%d)", + self.config.weight_type, + in_features // num_groups, + ) + return True + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: - """Repack CT GPTQ weights into the zentorch WOQ layout. + """Repack CT GPTQ weights into the zentorch DA8W4 or WOQ layout. Falls back to ``CPUWNA16LinearKernel.process_weights_after_loading`` via ``super()`` when the layer doesn't satisfy @@ -101,6 +193,9 @@ class ZentorchWNA16LinearKernel(CPUWNA16LinearKernel): if getattr(layer, "_zentorch_processed_weights", False): return + if self._maybe_process_da8w4_weights(layer): + return + if not self._zentorch_woq_eligible(layer): logger.info_once( "[zen_cpu] ZentorchWNA16 fast path not eligible for this " @@ -186,6 +281,16 @@ class ZentorchWNA16LinearKernel(CPUWNA16LinearKernel): x: torch.Tensor, bias: torch.Tensor | None = None, ) -> torch.Tensor: + if getattr(layer, "_zentorch_da8w4", False): + # The kernel reads bias through a raw pointer, so it must be + # contiguous; the weight and scales already are. + return torch.ops.zentorch.zentorch_dynamic_qlinear.default( + x if x.dtype == torch.bfloat16 else x.to(torch.bfloat16), + layer._zentorch_da8w4_packed, + layer._zentorch_da8w4_scale, + bias.contiguous() if bias is not None else None, + ) + if getattr(layer, "_zentorch_processed_weights", False): return torch.ops.zentorch.zentorch_woq_linear.default( x, diff --git a/vllm/model_executor/layers/fused_moe/experts/cpu_moe.py b/vllm/model_executor/layers/fused_moe/experts/cpu_moe.py index 3e7376fe90..9b66a1f853 100644 --- a/vllm/model_executor/layers/fused_moe/experts/cpu_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/cpu_moe.py @@ -51,6 +51,32 @@ from vllm.platforms import CpuArchEnum, current_platform from vllm.utils.math_utils import round_up logger = init_logger(__name__) +# =========================================================================== +# Weight layout +# =========================================================================== + + +def swigluoai_w13_interleave_perm( + experts_cls: type[mk.FusedMoEExperts], + activation: MoEActivation, + two_i: int, + device: torch.device, +) -> torch.Tensor | None: + """Index taking a half-split w13 ``[gate | up]`` to the interleaved order + ZenDNN's ``swiglu_oai_mul`` expects, or None when no reorder is needed. + Callers apply it along whichever axis holds their output channels.""" + if not getattr(experts_cls, "requires_interleaved_w13", False): + return None + if activation != MoEActivation.SWIGLUOAI: + return None + + i = two_i // 2 + return torch.stack( + [torch.arange(0, i, device=device), torch.arange(i, two_i, device=device)], + dim=1, + ).flatten() + + # =========================================================================== # Unquantized (BF16/FP16/FP32) MoE # =========================================================================== diff --git a/vllm/model_executor/layers/fused_moe/experts/zentorch_moe.py b/vllm/model_executor/layers/fused_moe/experts/zentorch_moe.py new file mode 100644 index 0000000000..0076b65966 --- /dev/null +++ b/vllm/model_executor/layers/fused_moe/experts/zentorch_moe.py @@ -0,0 +1,204 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Zentorch fused MoE experts for AMD Zen CPUs.""" + +from collections.abc import Callable + +import torch + +import vllm.model_executor.layers.fused_moe.modular_kernel as mk +from vllm.model_executor.kernels.linear.zentorch_utils import ( + _ZENTORCH_MOE_ACTIVATIONS, + _moe_activation_to_str, + has_zentorch_op, +) +from vllm.model_executor.layers.fused_moe.activation import MoEActivation +from vllm.model_executor.layers.fused_moe.config import ( + FusedMoEConfig, + FusedMoEParallelConfig, + FusedMoEQuantConfig, + RoutingMethodType, +) +from vllm.model_executor.layers.fused_moe.router.cpu_router import select_experts +from vllm.model_executor.layers.quantization.utils.quant_utils import ( + QuantKey, + kInt4Static, + kInt4Static32, +) + + +class ZentorchExpertsInt4(mk.FusedMoEExpertsMonolithic): + """DA8W4 (W4A8) group-quantized monolithic MoE experts.""" + + # swiglu_oai_mul reads gate/up interleaved, not in the half-split order the + # weight loader leaves behind. + requires_interleaved_w13 = True + + def __init__( + self, + moe_config: FusedMoEConfig, + quant_config: FusedMoEQuantConfig, + ): + super().__init__(moe_config, quant_config) + # Defaults hold until process_weights_after_loading sees the layer. + self.renormalize = moe_config.routing_method in ( + RoutingMethodType.Renormalize, + RoutingMethodType.RenormalizeNaive, + ) + self.custom_routing_function: Callable | None = None + + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: + """Capture the router config that the monolithic apply() signature + cannot carry; Custom routing has no other source for it.""" + self.renormalize = layer.renormalize + self.custom_routing_function = layer.custom_routing_function + + @property + def expects_unquantized_inputs(self) -> bool: + return True + + @staticmethod + def activation_format() -> mk.FusedMoEActivationFormat: + return mk.FusedMoEActivationFormat.Standard + + @staticmethod + def _supports_current_device() -> bool: + return has_zentorch_op(["zentorch_fused_moe", "zentorch_woq_repack_weight"]) + + @staticmethod + def is_supported_config( + cls: type[mk.FusedMoEExperts], + moe_config: FusedMoEConfig, + weight_key: QuantKey | None, + activation_key: QuantKey | None, + activation_format: mk.FusedMoEActivationFormat, + ) -> tuple[bool, str | None]: + supported, reason = mk.FusedMoEExperts.is_supported_config( + cls, moe_config, weight_key, activation_key, activation_format + ) + if not supported: + return supported, reason + if moe_config.in_dtype != torch.bfloat16: + return False, "kernel requires bfloat16 activations" + # The grouped GEMM needs tokens spread over at least two experts, which + # top_k=1 cannot satisfy once decode drops to a single token. + if moe_config.experts_per_token < 2: + return False, "kernel requires experts_per_token >= 2" + return True, None + + @staticmethod + def _supports_no_act_and_mul() -> bool: + return False + + @staticmethod + def _supports_activation(activation: MoEActivation) -> bool: + return _moe_activation_to_str(activation) in _ZENTORCH_MOE_ACTIVATIONS + + @staticmethod + def _supports_parallel_config( + moe_parallel_config: FusedMoEParallelConfig, + ) -> bool: + # apply() passes select_experts' global ids straight to the kernel, so + # every expert has to be resident on the rank. + return not moe_parallel_config.use_ep + + @staticmethod + def _supports_quant_scheme( + weight_key: QuantKey | None, + activation_key: QuantKey | None, + ) -> bool: + return (weight_key, activation_key) in [ + (kInt4Static, None), + (kInt4Static32, None), + ] + + @staticmethod + def _supports_routing_method( + routing_method: RoutingMethodType, + weight_key: QuantKey | None, + activation_key: QuantKey | None, + ) -> bool: + return routing_method in [ + RoutingMethodType.Default, + RoutingMethodType.Renormalize, + RoutingMethodType.RenormalizeNaive, + RoutingMethodType.Custom, + ] + + @staticmethod + def _supports_router_logits_dtype( + router_logits_dtype: torch.dtype | None, + routing_method: RoutingMethodType, + ) -> bool: + return True + + def supports_expert_map(self) -> bool: + return False + + def apply( + self, + hidden_states: torch.Tensor, + w1: torch.Tensor, + w2: torch.Tensor, + router_logits: torch.Tensor, + activation: MoEActivation, + global_num_experts: int, + expert_map: torch.Tensor | None, + a1q_scale: torch.Tensor | None, + apply_router_weight_on_input: bool, + # grouped topk + fused topk bias parameters + num_expert_group: int | None = None, + e_score_correction_bias: torch.Tensor | None = None, + routed_scaling_factor: float | None = None, + topk_group: int | None = None, + ) -> torch.Tensor: + if apply_router_weight_on_input: + raise NotImplementedError( + "ZentorchExpertsInt4 does not support " + "apply_router_weight_on_input=True." + ) + + if ( + self.moe_config.routing_method == RoutingMethodType.Custom + and self.custom_routing_function is None + ): + raise RuntimeError( + "ZentorchExpertsInt4 needs the model's custom_routing_function " + "for RoutingMethodType.Custom." + ) + + topk_weights, topk_ids = select_experts( + hidden_states=hidden_states, + router_logits=router_logits, + use_grouped_topk=num_expert_group is not None, + top_k=self.moe_config.experts_per_token, + renormalize=self.renormalize, + topk_group=topk_group, + num_expert_group=num_expert_group, + custom_routing_function=self.custom_routing_function, + scoring_func="softmax", + routed_scaling_factor=( + routed_scaling_factor if routed_scaling_factor is not None else 1.0 + ), + e_score_correction_bias=e_score_correction_bias, + ) + + output = torch.empty_like(hidden_states) + torch.ops.zentorch.zentorch_fused_moe( + output, + hidden_states, + w1, + w2, + self.w1_bias, + self.w2_bias, + topk_weights, + topk_ids, + False, # skip_weighted + _moe_activation_to_str(activation), + self.w1_scale, + self.w2_scale, + ) + return output + + +__all__ = ["ZentorchExpertsInt4"] diff --git a/vllm/model_executor/layers/fused_moe/oracle/int_wna16.py b/vllm/model_executor/layers/fused_moe/oracle/int_wna16.py index 791dc56766..239fe1d55a 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/int_wna16.py +++ b/vllm/model_executor/layers/fused_moe/oracle/int_wna16.py @@ -12,6 +12,7 @@ from compressed_tensors.quantization import ( import vllm._custom_ops as ops import vllm.model_executor.layers.fused_moe.modular_kernel as mk +from vllm import envs from vllm.config.kernel import MoEBackend from vllm.logger import init_logger from vllm.model_executor.layers.fused_moe.config import ( @@ -55,6 +56,7 @@ class WNA16MoEBackend(Enum): BATCHED_MARLIN = "BATCHED_MARLIN" HUMMING = "HUMMING" CPU = "CPU" + ZEN_CPU = "ZEN_CPU" FLASHINFER_TRTLLM = "FLASHINFER_TRTLLM" TRITON = "TRITON" XPU = "XPU" @@ -98,6 +100,12 @@ def backend_to_kernel_cls( ) return [CPUExpertsInt4] + elif backend == WNA16MoEBackend.ZEN_CPU: + from vllm.model_executor.layers.fused_moe.experts.zentorch_moe import ( + ZentorchExpertsInt4, + ) + + return [ZentorchExpertsInt4] elif backend == WNA16MoEBackend.EMULATION: from vllm.model_executor.layers.fused_moe.experts.int4_emulation_moe import ( Int4EmulationTritonExperts, @@ -117,7 +125,7 @@ def backend_to_kernel_cls( def _get_priority_backends() -> list[WNA16MoEBackend]: """Get available backends in priority order based on platform and config.""" if current_platform.is_cpu(): - return [WNA16MoEBackend.CPU] + return [WNA16MoEBackend.ZEN_CPU, WNA16MoEBackend.CPU] if current_platform.is_xpu(): return [WNA16MoEBackend.XPU] @@ -160,6 +168,27 @@ def _backend_incompatibility_reason( from vllm.model_executor.layers.quantization.auto_gptq import AutoGPTQConfig from vllm.model_executor.layers.quantization.moe_wna16 import MoeWNA16Config + if backend == WNA16MoEBackend.ZEN_CPU: + if not envs.VLLM_CPU_INT4_W4A8: + return "VLLM_CPU_INT4_W4A8=0 disables the DA8W4 path" + if may_have_zp: + return "zero points are not supported" + if isinstance(quant_config, MoeWNA16Config): + return "the MoeWNA16 weight layout is not supported" + if isinstance(quant_config, AutoGPTQConfig) and quant_config.desc_act: + return "GPTQ activation ordering is not supported" + if ( + isinstance(quant_config, QuantizationArgs) + and quant_config.actorder == "group" + ): + return "group activation ordering is not supported" + group_size = getattr(quant_config, "group_size", None) + if group_size is None or group_size <= 0: + return "DA8W4 requires group-quantized weights" + # AOCL sym_quant requires the group size to be a multiple of 4. + if group_size % 4 != 0: + return f"group size {group_size} is not a multiple of 4" + if backend == WNA16MoEBackend.TRITON: if may_have_bias: return "expert bias is not supported" @@ -403,10 +432,14 @@ def make_wna16_moe_kernel( from vllm.model_executor.layers.fused_moe.experts.xpu_moe import ( XPUExpertsWNA16, ) + from vllm.model_executor.layers.fused_moe.experts.zentorch_moe import ( + ZentorchExpertsInt4, + ) # Currently, we only support TrtLlmMxint4ExpertsMonolithic, MarlinExperts, - # BatchedMarlinExperts, XPUExpertsWNA16, CPUExpertsInt4, the Humming - # grouped/indexed experts, and Int4EmulationTritonExperts + # BatchedMarlinExperts, XPUExpertsWNA16, CPUExpertsInt4, + # ZentorchExpertsInt4, the Humming grouped/indexed experts, and + # Int4EmulationTritonExperts allowed_experts: tuple[type[mk.FusedMoEExperts], ...] = ( MarlinExperts, BatchedMarlinExperts, @@ -414,6 +447,7 @@ def make_wna16_moe_kernel( TrtLlmMxint4ExpertsMonolithic, XPUExpertsWNA16, CPUExpertsInt4, + ZentorchExpertsInt4, Int4EmulationTritonExperts, Rdna3WNA16Experts, ) @@ -1034,6 +1068,64 @@ def _process_weights_cpu( ) +def _zen_repack_s4(packed: torch.Tensor, in_features: int) -> torch.Tensor: + """Repack CT int4 ``[E, N, K//8]`` into zentorch s4 ``[E, N, K//8]``.""" + from vllm.model_executor.kernels.linear.mixed_precision.zentorch import ( + _import_unpack_from_int32, + ) + + num_experts, out_features, _ = packed.shape + # packed_dim indexes the 2D expert slice, which is N-first, so K is dim 1. + unpacked = _import_unpack_from_int32()( + packed, + 4, + torch.Size([num_experts, out_features, in_features]), + packed_dim=1, + ) + # CT is already N-first, which is what the repack consumes. + repack_op = torch.ops.zentorch.zentorch_woq_repack_weight.default + return torch.stack([repack_op(w) for w in unpacked]) + + +def _process_weights_zen_cpu( + w13: torch.Tensor, + w2: torch.Tensor, + w13_scale: torch.Tensor, + w2_scale: torch.Tensor, + w13_bias: torch.Tensor | None = None, + w2_bias: torch.Tensor | None = None, +) -> tuple[ + torch.Tensor, # w13_qweight + torch.Tensor, # w2_qweight + torch.Tensor, # w13_scales + torch.Tensor, # w2_scales + torch.Tensor | None, # w13_qzeros + torch.Tensor | None, # w2_qzeros + torch.Tensor | None, # w13_input_global_scale + torch.Tensor | None, # w2_input_global_scale + torch.Tensor | None, # w13_bias + torch.Tensor | None, # w2_bias +]: + """Zen CPU INT4 DA8W4 (W4A8) weight post-processing.""" + # CT WNA16 buffers are N-first, so the scales are [E, N, G]. + hidden_size = w2_scale.shape[1] + intermediate_size = w13_scale.shape[1] // 2 + + # The kernel reads per-group scales as [E, G, N], so N-first needs a swap. + return ( + _zen_repack_s4(w13.data, hidden_size), + _zen_repack_s4(w2.data, intermediate_size), + w13_scale.data.transpose(1, 2).to(torch.bfloat16).contiguous(), + w2_scale.data.transpose(1, 2).to(torch.bfloat16).contiguous(), + None, # w13_qzeros (symmetric) + None, # w2_qzeros (symmetric) + None, # w13_input_global_scale + None, # w2_input_global_scale + w13_bias, + w2_bias, + ) + + def _process_weights_xpu( layer: torch.nn.Module, quant_config: QuantizationConfig, @@ -1584,6 +1676,15 @@ def convert_to_wna16_moe_kernel_format( w13_bias, w2_bias, ) + elif backend == WNA16MoEBackend.ZEN_CPU: + return _process_weights_zen_cpu( + w13, + w2, + w13_scale, + w2_scale, + w13_bias, + w2_bias, + ) elif backend == WNA16MoEBackend.FLASHINFER_TRTLLM: return _process_weights_flashinfer( w13, diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_wna16.py b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_wna16.py index c03d2bfcf4..8bbaaecd64 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_wna16.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_wna16.py @@ -210,6 +210,27 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod): layer.register_parameter("w2_weight_packed", w2_weight) set_weight_attrs(w2_weight, extra_weight_attrs) + # Per-expert biases, as gpt-oss carries. + if self.moe.has_bias: + w13_num_shards = 2 if self.moe.is_act_and_mul else 1 + w13_bias = torch.nn.Parameter( + torch.zeros( + num_experts, + w13_num_shards * intermediate_size_per_partition, + dtype=params_dtype, + ), + requires_grad=False, + ) + layer.register_parameter("w13_bias", w13_bias) + set_weight_attrs(w13_bias, extra_weight_attrs) + + w2_bias = torch.nn.Parameter( + torch.zeros(num_experts, hidden_size, dtype=params_dtype), + requires_grad=False, + ) + layer.register_parameter("w2_bias", w2_bias) + set_weight_attrs(w2_bias, extra_weight_attrs) + if self.strategy == "channel": num_groups_w2 = num_groups_w13 = 1 self.group_size = -1 @@ -350,7 +371,41 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod): routing_tables=layer._expert_routing_tables(), ) + if self.wna16_backend == WNA16MoEBackend.ZEN_CPU: + # Monolithic experts route inside apply(), so like the unquantized + # CPU path they need the router config that only the layer carries. + self.moe_kernel.fused_experts.process_weights_after_loading(layer) + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: + from vllm.model_executor.layers.fused_moe.experts.cpu_moe import ( + swigluoai_w13_interleave_perm, + ) + + # Reorder before the conversion below, which repacks w13 for the kernel. + # The CT WNA16 buffers are N-first, so 2I is dim 1 of the packed weight + # [E, 2I, K/pack] and of its group scales [E, 2I, G], as for the bias. + perm = swigluoai_w13_interleave_perm( + self.experts_cls, + self.moe.activation, + layer.w13_weight_packed.size(1), + layer.w13_weight_packed.device, + ) + if perm is not None: + replace_parameter( + layer, + "w13_weight_packed", + layer.w13_weight_packed.data[:, perm].contiguous(), + ) + replace_parameter( + layer, + "w13_weight_scale", + layer.w13_weight_scale.data[:, perm].contiguous(), + ) + if self.moe.has_bias: + replace_parameter( + layer, "w13_bias", layer.w13_bias.data[:, perm].contiguous() + ) + # Process weights using the shared oracle infrastructure converted = convert_to_wna16_moe_kernel_format( backend=self.wna16_backend, @@ -363,6 +418,8 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod): w2_scale=layer.w2_weight_scale, w13_qzeros=getattr(layer, "w13_weight_zero_point", None), w2_qzeros=getattr(layer, "w2_weight_zero_point", None), + w13_bias=getattr(layer, "w13_bias", None), + w2_bias=getattr(layer, "w2_bias", None), ) if converted is None: @@ -378,8 +435,8 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod): w2_qzeros, w13_input_global_scale, w2_input_global_scale, - _, # w13_bias - _, # w2_bias + w13_bias, + w2_bias, ) = converted # Replace common parameters @@ -388,6 +445,11 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod): replace_parameter(layer, "w13_weight_scale", w13_scales) replace_parameter(layer, "w2_weight_scale", w2_scales) + if w13_bias is not None: + replace_parameter(layer, "w13_bias", w13_bias) + if w2_bias is not None: + replace_parameter(layer, "w2_bias", w2_bias) + # CPU fused_experts_cpu requires zero points even for symmetric quant. # EMULATION bakes ZP into the dequantized bf16 weights — ZP is None. if ( @@ -437,6 +499,8 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod): num_bits=self.num_bits, w1_zp=getattr(layer, "w13_weight_zero_point", None), w2_zp=getattr(layer, "w2_weight_zero_point", None), + w1_bias=getattr(layer, "w13_bias", None), + w2_bias=getattr(layer, "w2_bias", None), gemm1_clamp_limit=getattr(layer, "swiglu_limit", None), gemm1_alpha=getattr(layer, "swiglu_alpha", None), gemm1_beta=getattr(layer, "swiglu_beta", None), diff --git a/vllm/model_executor/models/gpt_oss.py b/vllm/model_executor/models/gpt_oss.py index 7c10aaa458..90286006a6 100644 --- a/vllm/model_executor/models/gpt_oss.py +++ b/vllm/model_executor/models/gpt_oss.py @@ -76,6 +76,8 @@ _GPT_OSS_STREAMED_EXPERT_SUFFIX_TO_SHARD = { "w2_bias": "gpt_oss_w2", "w13_weight_scale": "gpt_oss_w13", "w2_weight_scale": "gpt_oss_w2", + "w13_weight_packed": "gpt_oss_w13", + "w2_weight_packed": "gpt_oss_w2", } @@ -263,7 +265,7 @@ class GptOssRoutedExperts(RoutedExperts): ) -> None: tp_rank = self.moe_config.moe_parallel_config.tp_rank is_w13 = shard_id == "gpt_oss_w13" - is_weight = weight_name.endswith("_weight") + is_weight = weight_name.endswith(("_weight", "_weight_packed")) is_partitioned_scale = weight_name.endswith("_weight_scale") if is_weight: @@ -347,6 +349,7 @@ class GptOssRoutedExperts(RoutedExperts): ) or quant_method_name in ( "CompressedTensorsW4A4Nvfp4MoEMethod", "Nvfp4OnlineMoEMethod", + "CompressedTensorsWNA16MoEMethod", ): self._load_packed_expert(expert_data, loaded_weight, weight_name, shard_id) else: