vLLM/tests/kernels/moe/test_cpu_quant_fused_moe.py

# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for CPU quantized fused MoE kernels."""

import math
import sys

import pytest
import torch
import torch.nn.functional as F

from vllm.platforms import current_platform
from vllm.utils.torch_utils import set_random_seed

if not current_platform.is_cpu():
    pytest.skip("skipping CPU-only tests", allow_module_level=True)

import vllm._custom_ops as ops  # noqa: E402

if not hasattr(torch.ops._C, "fused_experts_cpu"):
    pytest.skip("fused_experts_cpu op not available", allow_module_level=True)


def _silu_and_mul(x: torch.Tensor) -> torch.Tensor:
    d = x.shape[-1] // 2
    return F.silu(x[..., :d]) * x[..., d:]


def _prepack_experts(w: torch.Tensor) -> torch.Tensor:
    """VNNI-prepack expert weights via ``convert_weight_packed``."""
    return torch.ops._C.convert_weight_packed(w)


def _deterministic_expert_routes(
    block_sizes: tuple[int, ...],
) -> tuple[torch.Tensor, torch.Tensor]:
    """Create top-1 routes with exact per-expert block occupancies."""
    topk_ids = torch.cat(
        [
            torch.full((size,), expert, dtype=torch.int32)
            for expert, size in enumerate(block_sizes)
        ]
    )
    topk_ids = topk_ids.view(-1, 1)
    topk_weights = torch.ones(topk_ids.shape, dtype=torch.float32)
    return topk_weights, topk_ids


# ===========================================================================
# FP8 W8A16 MoE
# ===========================================================================


BLOCK_SIZE = [128, 128]  # [block_n, block_k]

_FP8_INFO = torch.finfo(torch.float8_e4m3fn)
FP8_SCALE = _FP8_INFO.max  # 448.0
FACTOR_FOR_SCALE = 1e-3


def _block_dequant_weight(
    weight: torch.Tensor,
    scales: torch.Tensor,
    block_size: list[int],
) -> torch.Tensor:
    """Block-dequantize FP8 weight [E, N, K] -> float [E, N, K]."""
    E, N, K = weight.shape
    block_n, block_k = block_size
    pad_N = (block_n - N % block_n) % block_n
    pad_K = (block_k - K % block_k) % block_k

    if pad_N > 0 or pad_K > 0:
        weight = F.pad(weight, (0, pad_K, 0, pad_N))

    n_tiles = math.ceil(N / block_n)
    k_tiles = math.ceil(K / block_k)

    weight_block = (
        weight.view(E, n_tiles, block_n, k_tiles, block_k)
        .permute(0, 1, 3, 2, 4)
        .float()
        .contiguous()
    )
    weight_scaled = (
        (weight_block * scales.view(E, n_tiles, k_tiles, 1, 1))
        .permute(0, 1, 3, 2, 4)
        .contiguous()
    )
    if pad_N > 0 or pad_K > 0:
        weight_scaled = weight_scaled.view(E, N + pad_N, K + pad_K)
        weight_scaled = weight_scaled[..., :N, :K].contiguous()
    else:
        weight_scaled = weight_scaled.view(E, N, K)
    return weight_scaled


def ref_w8a16_block_fp8_moe(
    a: torch.Tensor,
    w1: torch.Tensor,
    w2: torch.Tensor,
    w1_s: torch.Tensor,
    w2_s: torch.Tensor,
    topk_weight: torch.Tensor,
    topk_ids: torch.Tensor,
    block_size: list[int],
) -> torch.Tensor:
    """Reference FP8 W8A16 block-scaled fused MoE in pure torch."""
    B, D = a.shape
    topk = topk_ids.size(1)

    w1_dq = _block_dequant_weight(w1, w1_s, block_size)
    w2_dq = _block_dequant_weight(w2, w2_s, block_size)

    a_exp = a.view(B, -1, D).repeat(1, topk, 1).reshape(-1, D).float()
    out = torch.zeros(B * topk, w2_dq.shape[1], dtype=torch.float32)

    topk_weight_flat = topk_weight.view(-1)
    topk_ids_flat = topk_ids.view(-1)

    for i in range(w1_dq.shape[0]):
        mask = topk_ids_flat == i
        if mask.sum():
            ic0 = torch.matmul(a_exp[mask], w1_dq[i].transpose(0, 1))
            ic1 = _silu_and_mul(ic0)
            out[mask] = torch.matmul(ic1, w2_dq[i].transpose(0, 1))

    return (
        (out.view(B, -1, w2_dq.shape[1]) * topk_weight_flat.view(B, -1, 1))
        .sum(dim=1)
        .to(a.dtype)
    )


def _make_fp8_moe_weights(
    E: int,
    N: int,
    K: int,
    block_size: list[int],
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
    """Generate random FP8 MoE weights with random block scales."""
    block_n, block_k = block_size

    w1 = (
        (torch.randn(E, 2 * N, K) * FP8_SCALE)
        .clamp(min=-FP8_SCALE, max=FP8_SCALE)
        .to(torch.float8_e4m3fn)
    )
    w2 = (
        (torch.randn(E, K, N) * FP8_SCALE)
        .clamp(min=-FP8_SCALE, max=FP8_SCALE)
        .to(torch.float8_e4m3fn)
    )

    w1_s = (
        torch.randn(E, math.ceil(2 * N / block_n), math.ceil(K / block_k))
        * FACTOR_FOR_SCALE
    )
    w2_s = (
        torch.randn(E, math.ceil(K / block_n), math.ceil(N / block_k))
        * FACTOR_FOR_SCALE
    )
    return w1, w2, w1_s, w2_s


FP8_NUM_TOKENS = [1, 2, 64, 121]
FP8_MOE_CONFIGS = [
    (256, 512, 8, 2),
    (256, 512, 8, 4),
    (512, 256, 8, 2),
    (512, 256, 8, 4),
    (512, 512, 8, 2),
    (512, 512, 8, 4),
    (768, 2048, 8, 2),
    (768, 2048, 8, 4),
    (768, 2048, 128, 8),
]


@pytest.mark.parametrize("M", FP8_NUM_TOKENS)
@pytest.mark.parametrize("N,K,E,topk", FP8_MOE_CONFIGS)
@pytest.mark.parametrize("seed", [0])
def test_w8a16_block_fp8_cpu_fused_moe(M, N, K, E, topk, seed):
    """Test fused_experts_cpu FP8 W8A16 against dequantised torch reference."""
    set_random_seed(seed)

    a = torch.randn(M, K, dtype=torch.bfloat16) / math.sqrt(K)
    w1, w2, w1_s, w2_s = _make_fp8_moe_weights(E, N, K, BLOCK_SIZE)

    score = torch.randn(M, E, dtype=torch.bfloat16)
    score = torch.softmax(score, dim=-1, dtype=torch.float32)
    topk_weight, topk_ids = torch.topk(score, topk)
    topk_ids = topk_ids.to(torch.int32)

    ref_out = ref_w8a16_block_fp8_moe(
        a, w1, w2, w1_s, w2_s, topk_weight, topk_ids, BLOCK_SIZE
    )

    pw1, pw2 = _prepack_experts(w1), _prepack_experts(w2)

    # Test writing into a fresh output buffer against reference
    out = torch.empty_like(a)
    ops.fused_experts_cpu(
        out,
        a.clone(),
        pw1,
        pw2,
        topk_weight,
        topk_ids,
        ops.CPUQuantMethod.FP8_W8A16,
        w1_s,
        w2_s,
        None,
        None,
        BLOCK_SIZE,
        is_vnni=True,
    )
    torch.testing.assert_close(ref_out.bfloat16(), out, atol=1e-2, rtol=1e-2)

    # Test writing into the input tensor itself (aliased in-place) produces
    # an identical result
    a_inplace = a.clone()
    ops.fused_experts_cpu(
        a_inplace,
        a_inplace,
        pw1,
        pw2,
        topk_weight,
        topk_ids,
        ops.CPUQuantMethod.FP8_W8A16,
        w1_s,
        w2_s,
        None,
        None,
        BLOCK_SIZE,
        is_vnni=True,
    )
    torch.testing.assert_close(a_inplace, out, atol=0, rtol=0)


def test_w8a16_block_fp8_cpu_fused_moe_small_expert_blocks():
    """Test FP8 BRGEMM at exact and multi-block expert boundaries."""
    set_random_seed(0)
    block_sizes = (4, 5, 33)
    N, K, E = 128, 128, len(block_sizes)
    M = sum(block_sizes)

    a = torch.randn(M, K, dtype=torch.bfloat16) / math.sqrt(K)
    w1, w2, w1_s, w2_s = _make_fp8_moe_weights(E, N, K, BLOCK_SIZE)
    topk_weight, topk_ids = _deterministic_expert_routes(block_sizes)

    ref_out = ref_w8a16_block_fp8_moe(
        a, w1, w2, w1_s, w2_s, topk_weight, topk_ids, BLOCK_SIZE
    )
    pw1, pw2 = _prepack_experts(w1), _prepack_experts(w2)
    out = torch.empty_like(a)
    ops.fused_experts_cpu(
        out,
        a,
        pw1,
        pw2,
        topk_weight,
        topk_ids,
        ops.CPUQuantMethod.FP8_W8A16,
        w1_s,
        w2_s,
        None,
        None,
        BLOCK_SIZE,
        is_vnni=True,
    )

    torch.testing.assert_close(ref_out.bfloat16(), out, atol=1e-2, rtol=1e-2)


# ===========================================================================
# MXFP4 W4A16 MoE
# ===========================================================================


class MXFP4QuantizeUtil:
    """MXFP4 quantization utility."""

    E2M1_max = 6.0
    E2M1_values = [0, 0.5, 1, 1.5, 2, 3, 4, 6]
    E2M1_bounds = torch.tensor([0.25, 0.75, 1.25, 1.75, 2.5, 3.5, 5])
    block_size = 32

    @classmethod
    def quantize(cls, input: torch.Tensor) -> tuple:
        """Quantize BF16 tensor to MXFP4 packed uint8 format."""

        def cast_fp4(x):
            sign = torch.sign(x)
            sign_bit = (2 - sign) // 2
            ord_ = torch.sum(
                (x.abs().unsqueeze(-1) - cls.E2M1_bounds.to(x.device)) > 0, dim=-1
            )
            fp4_val = (sign_bit * 0b1000 + ord_).to(torch.uint8)
            return fp4_val

        def fuse_uint4_to_uint8(x):
            left_side = x[..., 0::2]
            right_side = x[..., 1::2]
            new_data = right_side.clone() << 4
            new_data[..., : left_side.shape[-1]] += left_side
            return new_data

        original_shape = input.shape
        input = input.view(-1, cls.block_size)
        input_amax = input.abs().max(dim=-1, keepdim=True).values
        descale = input_amax / cls.E2M1_max
        min_value = torch.tensor(-127.0, device=descale.device)
        e8m0_scale = torch.ceil(torch.maximum(torch.log2(descale), min_value))

        input = (input / torch.exp2(e8m0_scale)).view(original_shape)
        input_q = cast_fp4(input)
        input_q = fuse_uint4_to_uint8(input_q)
        e8m0_scale = (e8m0_scale + 127).to(torch.uint8)
        return input_q, e8m0_scale

    @classmethod
    def dequantize(cls, quantized_data, dtype: torch.dtype, scale):
        """Dequantize MXFP4 packed tensor back to float."""

        def unfuse_uint8_to_uint4(x):
            left_side = x & 0x0F
            right_side = (x >> 4) & 0x0F
            shape = list(x.shape)
            shape[-1] = shape[-1] * 2
            result = torch.zeros(shape, dtype=torch.uint8, device=x.device)
            result[..., 0::2] = left_side
            result[..., 1::2] = right_side
            return result

        e8m0_scale = scale
        x_unfused = unfuse_uint8_to_uint4(quantized_data)
        sign = 1 - 2 * ((x_unfused & 0b1000) >> 3).to(torch.float32)
        magnitude = (x_unfused & 0b0111).to(torch.long)
        values = torch.tensor(cls.E2M1_values, device=quantized_data.device)
        original_shape = magnitude.shape
        x_float = values[magnitude.reshape(-1)].reshape(original_shape)
        x_float = sign.float() * x_float
        x_float = x_float.reshape(-1, cls.block_size)
        scale_factor = torch.exp2(e8m0_scale.float() - 127)
        scale_factor = scale_factor.reshape(-1, 1)
        x_float = x_float * scale_factor
        return x_float.reshape(original_shape).to(dtype)


def _swiglu(x: torch.Tensor, alpha: float, limit: float) -> torch.Tensor:
    """SwigLU activation used in GPT-OSS.
    Input is interleaved: [gate_0, up_0, gate_1, up_1, ...] in last dim.
    """
    gate = x[..., 0::2]
    up = x[..., 1::2]
    gate_clamped = torch.clamp(gate, max=limit)
    up_clamped = torch.clamp(up, min=-limit, max=limit)
    return gate_clamped * torch.sigmoid(alpha * gate_clamped) * (up_clamped + 1)


def ref_mxfp4_fused_moe(
    a: torch.Tensor,
    w1_dq: torch.Tensor,
    w2_dq: torch.Tensor,
    topk_weight: torch.Tensor,
    topk_ids: torch.Tensor,
    topk: int,
) -> torch.Tensor:
    """Reference MXFP4 fused MoE with SiLU activation."""
    B, D = a.shape
    a_f = a.float()
    out = torch.zeros(B * topk, w2_dq.shape[1], dtype=torch.float32)
    topk_ids_flat = topk_ids.view(-1)

    for i in range(w1_dq.shape[0]):
        mask = topk_ids_flat == i
        if mask.sum() == 0:
            continue
        token_indices = torch.where(mask)[0]
        source_indices = token_indices // topk
        ic0 = torch.matmul(a_f[source_indices], w1_dq[i].float().T)
        ic1 = _silu_and_mul(ic0)
        out[mask] = torch.matmul(ic1, w2_dq[i].float().T)

    return (out.view(B, topk, -1) * topk_weight.unsqueeze(-1)).sum(dim=1).to(a.dtype)


def ref_mxfp4_fused_moe_gptoss(
    a: torch.Tensor,
    w1_dq: torch.Tensor,
    w2_dq: torch.Tensor,
    w1_bias: torch.Tensor,
    w2_bias: torch.Tensor,
    topk_weight: torch.Tensor,
    topk_ids: torch.Tensor,
    alpha: float,
    limit: float,
) -> torch.Tensor:
    """Reference MXFP4 fused MoE with SwigLU+bias (GPT-OSS style)."""
    B, D = a.shape
    topk = topk_ids.shape[1]
    a_f = a.float()
    E = w1_dq.shape[0]
    out = torch.zeros(B * topk, w2_dq.shape[1], dtype=torch.float32)
    topk_ids_flat = topk_ids.view(-1)

    for i in range(E):
        mask = topk_ids_flat == i
        if mask.sum() == 0:
            continue
        token_indices = torch.where(mask)[0]
        source_indices = token_indices // topk
        ic0 = torch.matmul(a_f[source_indices], w1_dq[i].float().T)
        ic0 = ic0 + w1_bias[i].float()
        ic1 = _swiglu(ic0, alpha, limit)
        ic2 = torch.matmul(ic1, w2_dq[i].float().T)
        ic2 = ic2 + w2_bias[i].float()
        out[mask] = ic2

    return (out.view(B, topk, -1) * topk_weight.unsqueeze(-1)).sum(dim=1).to(a.dtype)


def _prepack_mxfp4_experts(
    w: torch.Tensor, w_scale: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor]:
    """VNNI-prepack MXFP4 weights and repack scales."""
    packed_w = torch.ops._C.convert_weight_packed(w)
    packed_s = torch.ops._C.convert_scale_packed(w_scale)
    return packed_w, packed_s


MXFP4_NUM_TOKENS = [1, 2, 32, 121]
MXFP4_MOE_CONFIGS = [
    (128, 128, 4, 2),
    (256, 256, 8, 4),
    (352, 256, 8, 4),
    (512, 320, 8, 4),
]


@pytest.mark.parametrize("M", MXFP4_NUM_TOKENS)
@pytest.mark.parametrize("N,K,E,topk", MXFP4_MOE_CONFIGS)
@pytest.mark.parametrize("seed", [0])
def test_mxfp4_cpu_fused_moe(M, N, K, E, topk, seed):
    """Test fused_experts_mxfp4_cpu against dequantized torch reference."""
    set_random_seed(seed)
    dtype = torch.bfloat16

    a = torch.randn(M, K, dtype=dtype) / 10

    # Generate and quantize weights
    w1_bf16 = torch.randn(E, 2 * N, K, dtype=dtype) / 10
    w1q, w1s = MXFP4QuantizeUtil.quantize(w1_bf16)
    w1s = w1s.reshape(E, 2 * N, K // 32)
    w1dq = MXFP4QuantizeUtil.dequantize(w1q, dtype, w1s)

    w2_bf16 = torch.randn(E, K, N, dtype=dtype) / 10
    w2q, w2s = MXFP4QuantizeUtil.quantize(w2_bf16)
    w2s = w2s.reshape(E, K, N // 32)
    w2dq = MXFP4QuantizeUtil.dequantize(w2q, dtype, w2s)

    # Routing
    score = torch.randn(M, E, dtype=dtype)
    score = torch.softmax(score, dim=-1, dtype=torch.float32)
    topk_weight, topk_ids = torch.topk(score, topk)
    topk_ids = topk_ids.to(torch.int32)

    # Reference
    ref_out = ref_mxfp4_fused_moe(a, w1dq, w2dq, topk_weight, topk_ids, topk)

    # Pack weights for kernel
    pw1, pw1s = _prepack_mxfp4_experts(w1q, w1s)
    pw2, pw2s = _prepack_mxfp4_experts(w2q, w2s)

    # Kernel
    a_in = a.clone()
    out = torch.empty_like(a_in)
    ops.fused_experts_cpu(
        out,
        a_in,
        pw1,
        pw2,
        topk_weight,
        topk_ids,
        ops.CPUQuantMethod.MXFP4,
        pw1s,  # w1_scale
        pw2s,  # w2_scale
        None,  # w1_zero
        None,  # w2_zero
        None,  # block_size
    )

    torch.testing.assert_close(ref_out.bfloat16(), out, atol=1e-2, rtol=1e-2)


def test_mxfp4_cpu_fused_moe_small_expert_blocks():
    """Test MXFP4 BRGEMM at exact and multi-block expert boundaries."""
    set_random_seed(0)
    block_sizes = (4, 5, 33)
    N, K, E = 128, 128, len(block_sizes)
    M = sum(block_sizes)
    dtype = torch.bfloat16

    a = torch.randn(M, K, dtype=dtype) / 10
    w1_bf16 = torch.randn(E, 2 * N, K, dtype=dtype) / 10
    w1q, w1s = MXFP4QuantizeUtil.quantize(w1_bf16)
    w1s = w1s.reshape(E, 2 * N, K // 32)
    w1dq = MXFP4QuantizeUtil.dequantize(w1q, dtype, w1s)

    w2_bf16 = torch.randn(E, K, N, dtype=dtype) / 10
    w2q, w2s = MXFP4QuantizeUtil.quantize(w2_bf16)
    w2s = w2s.reshape(E, K, N // 32)
    w2dq = MXFP4QuantizeUtil.dequantize(w2q, dtype, w2s)

    topk_weight, topk_ids = _deterministic_expert_routes(block_sizes)
    ref_out = ref_mxfp4_fused_moe(a, w1dq, w2dq, topk_weight, topk_ids, 1)

    pw1, pw1s = _prepack_mxfp4_experts(w1q, w1s)
    pw2, pw2s = _prepack_mxfp4_experts(w2q, w2s)
    out = torch.empty_like(a)
    ops.fused_experts_cpu(
        out,
        a,
        pw1,
        pw2,
        topk_weight,
        topk_ids,
        ops.CPUQuantMethod.MXFP4,
        pw1s,
        pw2s,
        None,
        None,
        None,
    )

    torch.testing.assert_close(ref_out.bfloat16(), out, atol=1e-2, rtol=1e-2)


# Both E2M1 zero codes: 0b0000 (+0.0) and 0b1000 (-0.0), in both nibbles.
MXFP4_ZERO_BYTES = [0x00, 0x88, 0x08, 0x80]
# 0 and 255 are the ends of the E8M0 range; 127 is the identity scale.
MXFP4_E8M0_VALUES = [0, 1, 127, 200, 254, 255]


@pytest.mark.parametrize("zero_byte", MXFP4_ZERO_BYTES)
@pytest.mark.parametrize("e8m0", MXFP4_E8M0_VALUES)
def test_mxfp4_cpu_zero_codes_stay_zero(zero_byte, e8m0):
    """A zero E2M1 code stays zero for every E8M0 exponent.

    The unpack applies the block scale as an integer add on the bf16 exponent
    field, which is exact for every value in the E2M1 codebook except the two
    zeros: 0x0000 and 0x8000 have no exponent to shift, so adding to them
    produces a small finite number instead of zero. Both are special-cased, and
    this is the invariant that special case exists for.

    Worth pinning separately from ``test_mxfp4_cpu_fused_moe``: there the zero
    codes are a small fraction of random weights and a broken special case
    would stay inside the 1e-2 tolerance for the low exponents. Here every
    weight is a zero, so the output is exactly zero or it is not.
    """
    N, K, E, M = 64, 64, 2, 4
    dtype = torch.bfloat16
    set_random_seed(0)

    a = torch.randn(M, K, dtype=dtype)
    w1q = torch.full((E, 2 * N, K // 2), zero_byte, dtype=torch.uint8)
    w1s = torch.full((E, 2 * N, K // 32), e8m0, dtype=torch.uint8)
    # w2 is ordinary: the zeros have to survive the first GEMM and the
    # activation, and a nonzero w2 is what would expose it if they did not.
    w2_bf16 = torch.randn(E, K, N, dtype=dtype) / 10
    w2q, w2s = MXFP4QuantizeUtil.quantize(w2_bf16)
    w2s = w2s.reshape(E, K, N // 32)

    topk_weight = torch.ones((M, 1), dtype=torch.float32)
    topk_ids = torch.zeros((M, 1), dtype=torch.int32)

    pw1, pw1s = _prepack_mxfp4_experts(w1q, w1s)
    pw2, pw2s = _prepack_mxfp4_experts(w2q, w2s)
    out = torch.empty_like(a)
    ops.fused_experts_cpu(
        out,
        a,
        pw1,
        pw2,
        topk_weight,
        topk_ids,
        ops.CPUQuantMethod.MXFP4,
        pw1s,
        pw2s,
        None,
        None,
        None,
    )

    # silu(0) * 0 = 0, so the whole layer collapses to exactly zero. Not
    # assert_close: any nonzero output here is a wrong unpack, not rounding.
    assert torch.equal(out, torch.zeros_like(out)), (
        f"zero code 0x{zero_byte:02x} with e8m0={e8m0} produced "
        f"max |out| = {out.abs().max().item()}"
    )


# Narrower than MXFP4_E8M0_VALUES on purpose: this test keeps nonzero weights,
# and 6.0 * 2**(255-127) is not representable in bf16, so the ends of the E8M0
# range would compare inf against inf and prove nothing. The all-zero test above
# is the one that can reach them.
MXFP4_E8M0_FINITE = [107, 127, 137]


@pytest.mark.parametrize("e8m0", MXFP4_E8M0_FINITE)
def test_mxfp4_cpu_zero_codes_mixed_with_nonzero(e8m0):
    """Zeros and nonzeros in the same 32-element scale block.

    The zero check is per lane, not per block: this fails if it is ever
    rewritten as a whole-block branch. Uses a single shared exponent so the
    reference is a plain power of two.
    """
    N, K, E, M = 64, 64, 2, 4
    dtype = torch.bfloat16
    set_random_seed(0)

    a = torch.randn(M, K, dtype=dtype)
    # Alternate a zero byte and a nonzero one along K, so every scale block
    # holds both kinds.
    pattern = torch.tensor([0x88, 0x21], dtype=torch.uint8).repeat(K // 4)
    w1q = pattern.view(1, 1, -1).expand(E, 2 * N, K // 2).contiguous()
    w1s = torch.full((E, 2 * N, K // 32), e8m0, dtype=torch.uint8)
    w1dq = MXFP4QuantizeUtil.dequantize(w1q, dtype, w1s)

    w2_bf16 = torch.randn(E, K, N, dtype=dtype) / 10
    w2q, w2s = MXFP4QuantizeUtil.quantize(w2_bf16)
    w2s = w2s.reshape(E, K, N // 32)
    w2dq = MXFP4QuantizeUtil.dequantize(w2q, dtype, w2s)

    topk_weight = torch.ones((M, 1), dtype=torch.float32)
    topk_ids = torch.zeros((M, 1), dtype=torch.int32)
    ref_out = ref_mxfp4_fused_moe(a, w1dq, w2dq, topk_weight, topk_ids, 1)

    pw1, pw1s = _prepack_mxfp4_experts(w1q, w1s)
    pw2, pw2s = _prepack_mxfp4_experts(w2q, w2s)
    out = torch.empty_like(a)
    ops.fused_experts_cpu(
        out,
        a,
        pw1,
        pw2,
        topk_weight,
        topk_ids,
        ops.CPUQuantMethod.MXFP4,
        pw1s,
        pw2s,
        None,
        None,
        None,
    )

    torch.testing.assert_close(ref_out.bfloat16(), out, atol=1e-2, rtol=1e-2)


@pytest.mark.parametrize("M", [1, 32])
@pytest.mark.parametrize("N,K,E,topk", [(128, 128, 4, 2), (64, 64, 4, 2)])
@pytest.mark.parametrize("seed", [0])
def test_mxfp4_cpu_fused_moe_bias_swiglu(M, N, K, E, topk, seed):
    """Test fused_experts_mxfp4_cpu with bias and SwigLU activation (GPT-OSS)."""
    set_random_seed(seed)
    dtype = torch.bfloat16
    alpha = 1.702
    limit = 7.0

    a = torch.randn(M, K, dtype=dtype) / 10

    # Generate and quantize weights
    w1_bf16 = torch.randn(E, 2 * N, K, dtype=dtype) / 10
    w1q, w1s = MXFP4QuantizeUtil.quantize(w1_bf16)
    w1s = w1s.reshape(E, 2 * N, K // 32)
    w1dq = MXFP4QuantizeUtil.dequantize(w1q, dtype, w1s)
    w1_b = torch.randn(E, 2 * N, dtype=torch.float32) / 10

    w2_bf16 = torch.randn(E, K, N, dtype=dtype) / 10
    w2q, w2s = MXFP4QuantizeUtil.quantize(w2_bf16)
    w2s = w2s.reshape(E, K, N // 32)
    w2dq = MXFP4QuantizeUtil.dequantize(w2q, dtype, w2s)
    w2_b = torch.randn(E, K, dtype=torch.float32) / 10

    # Routing
    score = torch.randn(M, E, dtype=dtype)
    score = torch.softmax(score, dim=-1, dtype=torch.float32)
    topk_weight, topk_ids = torch.topk(score, topk)
    topk_ids = topk_ids.to(torch.int32)

    # Reference
    ref_out = ref_mxfp4_fused_moe_gptoss(
        a, w1dq, w2dq, w1_b, w2_b, topk_weight, topk_ids, alpha, limit
    )

    # Pack weights for kernel
    pw1, pw1s = _prepack_mxfp4_experts(w1q, w1s)
    pw2, pw2s = _prepack_mxfp4_experts(w2q, w2s)

    # Kernel
    a_in = a.clone()
    out = torch.empty_like(a_in)
    ops.fused_experts_cpu(
        out,
        a_in,
        pw1,
        pw2,
        topk_weight,
        topk_ids,
        ops.CPUQuantMethod.MXFP4,
        pw1s,  # w1_scale
        pw2s,  # w2_scale
        None,  # w1_zero
        None,  # w2_zero
        None,  # block_size
        w1_bias=w1_b,
        w2_bias=w2_b,
        alpha=alpha,
        limit=limit,
    )

    torch.testing.assert_close(ref_out.bfloat16(), out, atol=1e-2, rtol=1e-2)


# ===========================================================================
# INT4 W4A16 MoE
# ===========================================================================


def _pack_int4_gptq(w_int4: torch.Tensor) -> torch.Tensor:
    """Pack INT4 values [N, K] → [N, K//8] int32 along K dim (GPTQ format)."""
    N, K = w_int4.shape
    assert K % 8 == 0
    w = w_int4.to(torch.int32)
    w_packed = torch.zeros(N, K // 8, dtype=torch.int32)
    for j in range(8):
        w_packed |= (w[:, j::8] & 0xF) << (j * 4)
    return w_packed


def _pack_int4_awq(w_int4: torch.Tensor) -> torch.Tensor:
    """Pack INT4 values [..., N] → [..., N//8] int32 along last dim (AWQ format)."""
    # AWQ packing bitshifts: indices {0,4,1,5,2,6,3,7} * 4 bits each
    _AWQ_BITSHIFTS = [0, 16, 4, 20, 8, 24, 12, 28]

    N = w_int4.shape[-1]
    assert N % 8 == 0
    w = w_int4.to(torch.int32)
    w_packed = torch.zeros(*w.shape[:-1], N // 8, dtype=torch.int32)
    for j, shift in enumerate(_AWQ_BITSHIFTS):
        w_packed |= (w[..., j::8] & 0xF) << shift
    return w_packed


def _ref_int4_moe(
    a: torch.Tensor,
    w1_int4: torch.Tensor,
    w2_int4: torch.Tensor,
    w1_zeros: torch.Tensor | None,
    w2_zeros: torch.Tensor | None,
    w1_s: torch.Tensor,
    w2_s: torch.Tensor,
    topk_weight: torch.Tensor,
    topk_ids: torch.Tensor,
    group_size: int,
) -> torch.Tensor:
    """Reference INT4 W4A16 group-quantized fused MoE in pure torch."""
    B = a.shape[0]
    topk = topk_ids.size(1)
    K_out = a.shape[1]

    out = torch.zeros(B, topk, K_out, dtype=torch.float32)
    for b in range(B):
        for t in range(topk):
            eid = topk_ids[b, t].item()
            x = a[b : b + 1].float()

            # Dequantize w1: [K, 2*N], groups along K (input dim)
            K_dim = w1_int4.shape[1]
            w1_dq = torch.zeros(K_dim, w1_int4.shape[2], dtype=torch.float32)
            for g in range(w1_s.shape[1]):
                k_start = g * group_size
                k_end = min((g + 1) * group_size, K_dim)
                zp = w1_zeros[eid, g, :].float() if w1_zeros is not None else 8.0
                w1_dq[k_start:k_end, :] = (
                    w1_int4[eid, k_start:k_end, :].float() - zp
                ) * w1_s[eid, g, :].float()

            ic = torch.matmul(x, w1_dq)  # [1, K] @ [K, 2*N] → [1, 2*N]
            ic = _silu_and_mul(ic)  # [1, N]

            # Dequantize w2: [N, K], groups along N (input dim)
            N_dim = w2_int4.shape[1]
            w2_dq = torch.zeros(N_dim, w2_int4.shape[2], dtype=torch.float32)
            for g in range(w2_s.shape[1]):
                n_start = g * group_size
                n_end = min((g + 1) * group_size, N_dim)
                zp = w2_zeros[eid, g, :].float() if w2_zeros is not None else 8.0
                w2_dq[n_start:n_end, :] = (
                    w2_int4[eid, n_start:n_end, :].float() - zp
                ) * w2_s[eid, g, :].float()

            oc = torch.matmul(ic, w2_dq)  # [1, N] @ [N, K] → [1, K]
            out[b, t] = oc.squeeze(0)

    return (out * topk_weight.unsqueeze(-1)).sum(dim=1).to(a.dtype)


def _make_int4_moe_weights(E, N, K, group_size, quant_algo):
    """Create INT4 MoE weights in GPTQ or AWQ packed format.

    Canonical layout (input × output):
      w1_int4: [E, K, 2*N]  w2_int4: [E, N, K]

    GPTQ packed (pack transposed weight along input/K dim):
      w1_packed: [E, K//8, 2*N]  w2_packed: [E, N//8, K]
      zeros: actual int4 zero points, same packing as weights

    AWQ packed (pack along output/N dim):
      w1_packed: [E, K, 2*N//8]  w2_packed: [E, N, K//8]
      zeros: actual int4 zero points, same packing as weights

    Returns:
        w1_int4, w2_int4,
        w1_packed, w2_packed,
        w1_zeros, w2_zeros,
        w1_zeros_packed, w2_zeros_packed,
        w1_s, w2_s

    """
    w1_int4 = torch.randint(0, 16, (E, K, 2 * N), dtype=torch.int32)
    w2_int4 = torch.randint(0, 16, (E, N, K), dtype=torch.int32)

    num_groups_w1 = K // group_size
    num_groups_w2 = N // group_size
    w1_s = (
        torch.randn(E, num_groups_w1, 2 * N, dtype=torch.bfloat16) * 0.01
    ).abs() + 0.001
    w2_s = (torch.randn(E, num_groups_w2, K, dtype=torch.bfloat16) * 0.01).abs() + 0.001

    if quant_algo == ops.CPUQuantAlgo.GPTQ:
        # Pack: canonical [E, K, 2*N] → transpose [E, 2*N, K] → GPTQ pack
        # [E, 2*N, K//8] → transpose [E, K//8, 2*N]
        w1_t = w1_int4.transpose(1, 2).contiguous()  # [E, 2*N, K]
        w1_packed = (
            torch.stack([_pack_int4_gptq(w1_t[e]) for e in range(E)])
            .transpose(1, 2)
            .contiguous()
        )  # [E, K//8, 2*N]
        w2_t = w2_int4.transpose(1, 2).contiguous()  # [E, K, N]
        w2_packed = (
            torch.stack([_pack_int4_gptq(w2_t[e]) for e in range(E)])
            .transpose(1, 2)
            .contiguous()
        )  # [E, N//8, K]
        w1_zeros = w2_zeros = None
        w1_zeros_packed = torch.full(
            (E, num_groups_w1, 2 * N // 8), 0x77777777, dtype=torch.int32
        )
        w2_zeros_packed = torch.full(
            (E, num_groups_w2, K // 8), 0x77777777, dtype=torch.int32
        )
    else:  # AWQ
        # Asymmetric: actual zero points, packed along output dim.
        w1_zeros = torch.randint(1, 15, (E, num_groups_w1, 2 * N), dtype=torch.int32)
        w2_zeros = torch.randint(1, 15, (E, num_groups_w2, K), dtype=torch.int32)
        w1_packed = torch.stack(
            [_pack_int4_awq(w1_int4[e]) for e in range(E)]
        )  # [E, K, 2*N//8]
        w2_packed = torch.stack(
            [_pack_int4_awq(w2_int4[e]) for e in range(E)]
        )  # [E, N, K//8]
        w1_zeros_packed = torch.stack(
            [_pack_int4_awq(w1_zeros[e]) for e in range(E)]
        )  # [E, K//gs, 2*N//8]
        w2_zeros_packed = torch.stack(
            [_pack_int4_awq(w2_zeros[e]) for e in range(E)]
        )  # [E, N//gs, K//8]

    return (
        w1_int4,
        w2_int4,
        w1_packed,
        w2_packed,
        w1_zeros,
        w2_zeros,
        w1_zeros_packed,
        w2_zeros_packed,
        w1_s,
        w2_s,
    )


INT4_MOE_CONFIGS = [
    # (N, K, E, topk, group_size)
    (256, 512, 8, 2, 128),
    (512, 256, 8, 2, 128),
    (512, 512, 8, 4, 128),
    (768, 2048, 8, 2, 128),
]


@pytest.mark.parametrize("M", [1, 2, 64, 121])
@pytest.mark.parametrize("N,K,E,topk,group_size", INT4_MOE_CONFIGS)
@pytest.mark.parametrize("quant_algo", [ops.CPUQuantAlgo.GPTQ, ops.CPUQuantAlgo.AWQ])
@pytest.mark.parametrize("seed", [0])
def test_int4_w4a16_cpu_fused_moe(M, N, K, E, topk, group_size, quant_algo, seed):
    """Test fused_experts_cpu INT4 W4A16 for both GPTQ and AWQ quant formats."""
    set_random_seed(seed)

    a = torch.randn(M, K, dtype=torch.bfloat16) / (0.5 * K**0.5)
    (
        w1_int4,
        w2_int4,
        w1_packed,
        w2_packed,
        w1_zeros,
        w2_zeros,
        w1_zeros_packed,
        w2_zeros_packed,
        w1_s,
        w2_s,
    ) = _make_int4_moe_weights(E, N, K, group_size, quant_algo)

    score = torch.randn(M, E, dtype=torch.bfloat16)
    score = torch.softmax(score, dim=-1, dtype=torch.float32)
    topk_weight, topk_ids = torch.topk(score, topk)
    topk_ids = topk_ids.to(torch.int32)

    ref_out = _ref_int4_moe(
        a,
        w1_int4,
        w2_int4,
        w1_zeros,
        w2_zeros,
        w1_s,
        w2_s,
        topk_weight,
        topk_ids,
        group_size,
    )

    from vllm.model_executor.layers.fused_moe.experts.cpu_moe import (
        prepare_int4_moe_layer_for_cpu,
    )

    (blocked_w1, blocked_w2, blocked_s1, blocked_s2, blocked_z1, blocked_z2) = (
        prepare_int4_moe_layer_for_cpu(
            w1_packed,
            w2_packed,
            w1_s,
            w2_s,
            quant_algo=quant_algo,
            w13_zeros=w1_zeros_packed,
            w2_zeros=w2_zeros_packed,
        )
    )

    a_in = a.clone()
    out = torch.empty_like(a_in)
    ops.fused_experts_cpu(
        out,
        a_in,
        blocked_w1,
        blocked_w2,
        topk_weight,
        topk_ids,
        ops.CPUQuantMethod.INT4_W4A8,
        blocked_s1,
        blocked_s2,
        blocked_z1,
        blocked_z2,
        None,  # block_size
        None,  # w1_bias
        None,  # w2_bias
        None,  # alpha
        None,  # limit
        True,  # is_vnni
    )
    torch.testing.assert_close(ref_out.bfloat16(), out, atol=1e-2, rtol=1e-2)


# ===========================================================================
# INT8 W8A8 MoE
# ===========================================================================


def _quantize_per_channel(w):
    """Symmetric per-channel INT8 quantisation. w: [N, K] -> (int8, scale)."""
    amax = w.abs().amax(dim=-1, keepdim=True).clamp(min=1e-12)
    scale = amax / 127.0
    w_q = (w / scale).round().clamp(-128, 127).to(torch.int8)
    return w_q, scale.float()


def _quantize_per_token(x):
    """Symmetric per-token INT8 quantisation. x: [M, K] -> (int8, scale)."""
    amax = x.abs().amax(dim=-1, keepdim=True).clamp(min=1e-12)
    scale = amax / 127.0
    x_q = (x / scale).round().clamp(-128, 127).to(torch.int8)
    return x_q, scale.float()


def _ref_int8_moe(a, w1, w2, w1_s, w2_s, topk_weight, topk_ids):
    """Reference INT8 W8A8 per-channel fused MoE in pure torch."""
    B, D = a.shape
    topk = topk_ids.size(1)

    out = torch.zeros(B, topk, w2.shape[1], dtype=torch.float32)
    for b in range(B):
        for t in range(topk):
            eid = topk_ids[b, t].item()

            x = a[b : b + 1].float()
            x_q, x_s = _quantize_per_token(x)
            ic = torch.matmul(x_q.float(), w1[eid].float().t())
            ic = ic * x_s * w1_s[eid].view(1, -1)
            ic = _silu_and_mul(ic)

            ic_q, ic_s = _quantize_per_token(ic)
            oc = torch.matmul(ic_q.float(), w2[eid].float().t())
            oc = oc * ic_s * w2_s[eid].view(1, -1)
            out[b, t] = oc.squeeze(0)

    result = (out * topk_weight.unsqueeze(-1)).sum(dim=1)
    return result.to(a.dtype)


def _make_int8_moe_weights(E, N, K):
    factor = 1e-2
    w1_f = (torch.randn(E, 2 * N, K) - 0.5) * 2
    w2_f = (torch.randn(E, K, N) - 0.5) * 2

    w1_q_list, w1_s_list = [], []
    w2_q_list, w2_s_list = [], []
    for e in range(E):
        q, s = _quantize_per_channel(w1_f[e])
        w1_q_list.append(q)
        w1_s_list.append(s)
        q, s = _quantize_per_channel(w2_f[e])
        w2_q_list.append(q)
        w2_s_list.append(s)

    return (
        torch.stack(w1_q_list),
        torch.stack(w2_q_list),
        torch.stack(w1_s_list) * factor,
        torch.stack(w2_s_list) * factor,
    )


INT8_NUM_TOKENS = [1, 2, 64, 121]
INT8_MOE_CONFIGS = [
    # (N, K, E, topk)
    (256, 512, 8, 2),
    (512, 256, 8, 2),
    (512, 512, 8, 4),
    (768, 2048, 8, 2),
]


@pytest.mark.parametrize("M", INT8_NUM_TOKENS)
@pytest.mark.parametrize("N,K,E,topk", INT8_MOE_CONFIGS)
@pytest.mark.parametrize("seed", [0])
@pytest.mark.parametrize("is_vnni", [False, True])
@pytest.mark.parametrize("inplace", [False, True])
def test_int8_w8a8_cpu_fused_moe(M, N, K, E, topk, seed, is_vnni, inplace):
    """Test fused_experts_cpu INT8 W8A8 against torch reference.

    ``inplace`` exercises writing into a fresh output buffer vs. aliasing
    the input tensor as the output buffer.
    """
    set_random_seed(seed)

    a = torch.randn(M, K, dtype=torch.bfloat16) / (0.5 * K**0.5)
    w1_q, w2_q, w1_s, w2_s = _make_int8_moe_weights(E, N, K)

    score = torch.randn(M, E, dtype=torch.bfloat16)
    score = torch.softmax(score, dim=-1, dtype=torch.float32)
    topk_weight, topk_ids = torch.topk(score, topk)
    topk_ids = topk_ids.to(torch.int32)

    ref_out = _ref_int8_moe(a, w1_q, w2_q, w1_s, w2_s, topk_weight, topk_ids)

    w1 = _prepack_experts(w1_q) if is_vnni else w1_q
    w2 = _prepack_experts(w2_q) if is_vnni else w2_q

    a_in = a.clone()
    out = a_in if inplace else torch.empty_like(a_in)
    ops.fused_experts_cpu(
        out,
        a_in,
        w1,
        w2,
        topk_weight,
        topk_ids,
        ops.CPUQuantMethod.INT8_W8A8,
        w1_s,
        w2_s,
        None,  # w1_zero
        None,  # w2_zero
        None,  # block_size
        None,  # w1_bias
        None,  # w2_bias
        None,  # alpha
        None,  # limit
        is_vnni,
    )
    torch.testing.assert_close(
        ref_out.bfloat16(),
        out,
        atol=2e-1,
        rtol=2e-1,
    )


# ===========================================================================
# FP8 W8A8 MoE
# ===========================================================================


requires_cpu_fp8_w8a8 = pytest.mark.skipif(
    not ops.cpu_has_amx_fp8(),
    reason="requires native AMX-FP8 hardware",
)

FP8_W8A8_MAX = torch.finfo(torch.float8_e4m3fn).max
FP8_W8A8_QUANT_GROUP = 128  # quantization group size for both K and N dimensions


def _make_fp8_w8a8_weight_w13(
    E: int, two_n: int, K: int, group_K: int = FP8_W8A8_QUANT_GROUP
) -> tuple[torch.Tensor, torch.Tensor]:
    """Create w13 [E, 2N, K] FP8 with group scales [E, 2N, G] where G = K//group_K."""
    G = K // group_K
    w_list, s_list = [], []
    for _ in range(E):
        w_f32 = torch.randn(two_n, K)
        # Per-row, per-K-group quantization: scale [2N, G]
        w_re = w_f32.view(two_n, G, group_K)
        abs_max = w_re.abs().amax(dim=2, keepdim=True).clamp(min=1e-7)  # [2N, G, 1]
        scale = (abs_max / FP8_W8A8_MAX).squeeze(2)  # [2N, G]
        w_q = (
            (w_re / abs_max).clamp(-FP8_W8A8_MAX, FP8_W8A8_MAX).to(torch.float8_e4m3fn)
        )
        w_list.append(w_q.view(two_n, K).contiguous())
        s_list.append(scale.float())
    return torch.stack(w_list), torch.stack(s_list)  # [E, 2N, K], [E, 2N, G]


def _make_fp8_w8a8_weight_w2(
    E: int,
    K: int,
    N: int,
    group_K: int = FP8_W8A8_QUANT_GROUP,
    group_N: int = FP8_W8A8_QUANT_GROUP,
) -> tuple[torch.Tensor, torch.Tensor]:
    """Create w2 [E, K, N] FP8 with block scales [E, K//gK, N//gN]."""
    nK = K // group_K
    nN = N // group_N
    w_list, s_list = [], []
    for _ in range(E):
        w_f32 = torch.randn(K, N)
        w_re = w_f32.view(nK, group_K, nN, group_N)
        abs_max = w_re.abs().amax(dim=(1, 3), keepdim=True).clamp(min=1e-7)
        scale = (abs_max / FP8_W8A8_MAX).squeeze(1).squeeze(2)  # [nK, nN]
        w_q = (
            (w_re / abs_max).clamp(-FP8_W8A8_MAX, FP8_W8A8_MAX).to(torch.float8_e4m3fn)
        )
        w_list.append(w_q.view(K, N).contiguous())
        s_list.append(scale.float())
    return torch.stack(w_list), torch.stack(s_list)  # [E, K, N], [E, K//gK, N//gN]


def _pack_fp8_w8a8_w13(
    w13: torch.Tensor,  # [E, 2N, K] FP8
    w13_scale: torch.Tensor,  # [E, 2N, G] float32
) -> tuple[torch.Tensor, torch.Tensor]:
    """Pack w13 for CPU FP8 W8A8 MoE kernel using float8_linear_prepack_cpu."""
    E = w13.size(0)
    packed_list, scale_list = [], []
    for i in range(E):
        pw, ps = torch.ops._C.float8_linear_prepack_cpu(
            w13[i].contiguous(), w13_scale[i].contiguous()
        )
        packed_list.append(pw)
        scale_list.append(ps)
    return torch.stack(packed_list), torch.stack(scale_list)


def _ref_fp8_w8a8_moe(
    hidden_states: torch.Tensor,  # BF16 [M, K_hidden] (after FP8 round-trip dequant)
    w13: torch.Tensor,  # FP8 [E, 2N, K_hidden]
    w13_scale: torch.Tensor,  # float32 [E, 2N, G] (G = K_hidden//group_K)
    w2: torch.Tensor,  # FP8 [E, K_hidden, N]
    w2_scale: torch.Tensor,  # float32 [E, K_hidden//gK, N//gN]
    topk_weights: torch.Tensor,  # float32 [M, top_k]
    topk_ids: torch.Tensor,  # int64 [M, top_k]
    N: int,
    group_K: int = FP8_W8A8_QUANT_GROUP,
    group_N: int = FP8_W8A8_QUANT_GROUP,
) -> torch.Tensor:
    """Reference implementation: dequantize weights then compute MoE."""
    M, K_hidden = hidden_states.shape
    top_k = topk_ids.shape[1]
    output = torch.zeros(M, K_hidden, dtype=torch.bfloat16)

    for tok in range(M):
        x = hidden_states[tok].float()  # [K_hidden]
        for k_idx in range(top_k):
            expert_id = topk_ids[tok, k_idx].item()
            w = topk_weights[tok, k_idx].item()

            # Stage 1: x @ w13.T — gate and up projections
            w13_e = w13[expert_id].float()  # [2N, K_hidden]
            ws_e = w13_scale[expert_id].float()  # [2N, G]
            G = ws_e.shape[1]
            w13_dq = torch.zeros_like(w13_e)
            for g in range(G):
                c0, c1 = g * group_K, (g + 1) * group_K
                w13_dq[:, c0:c1] = w13_e[:, c0:c1] * ws_e[:, g : g + 1]
            gate_up = (x.unsqueeze(0) @ w13_dq.T).squeeze(0)  # [2N]

            gate = gate_up[:N]
            up = gate_up[N:]
            act = F.silu(gate) * up  # [N]

            # Stage 2: act @ w2.T — down projection
            w2_e = w2[expert_id].float()  # [K_hidden, N]
            ws2_e = w2_scale[expert_id].float()  # [K_hidden//gK, N//gN]
            nK2 = K_hidden // group_K
            nN2 = N // group_N
            w2_dq = torch.zeros_like(w2_e)
            for gi in range(nK2):
                for gj in range(nN2):
                    r0, r1 = gi * group_K, (gi + 1) * group_K
                    c0, c1 = gj * group_N, (gj + 1) * group_N
                    w2_dq[r0:r1, c0:c1] = w2_e[r0:r1, c0:c1] * ws2_e[gi, gj]
            out_e = (act.unsqueeze(0) @ w2_dq.T).squeeze(0)  # [K_hidden]

            output[tok] += (w * out_e).to(torch.bfloat16)

    return output


@requires_cpu_fp8_w8a8
@pytest.mark.parametrize(
    "E,N,K,M,top_k",
    [
        # N and K must be multiples of FP8_W8A8_QUANT_GROUP=128
        (4, 128, 256, 1, 2),  # single token (decode phase)
        (4, 128, 256, 8, 2),
        (8, 256, 512, 16, 2),
        (4, 256, 512, 4, 1),
    ],
)
def test_fp8_w8a8_cpu_fused_moe(E: int, N: int, K: int, M: int, top_k: int):
    """Test fused_experts_cpu FP8_W8A8 shape/dtype and accuracy vs a
    dequantized BF16 reference."""
    set_random_seed(42)

    w13, w13_scale = _make_fp8_w8a8_weight_w13(E, 2 * N, K)
    w2, w2_scale = _make_fp8_w8a8_weight_w2(E, K, N)

    packed_w13, packed_w13_scale = _pack_fp8_w8a8_w13(w13, w13_scale)
    packed_w2 = _prepack_experts(w2)

    hidden_states = torch.randn(M, K, dtype=torch.bfloat16) * 0.1
    x_fp8, x_scales = torch.ops._C.quantize_fp8e4m3_vec(hidden_states, True, None)
    assert x_fp8.dtype == torch.float8_e4m3fn
    assert x_scales.shape == (M,)

    gen = torch.Generator().manual_seed(0)
    topk_weights = torch.softmax(torch.randn(M, top_k, generator=gen), dim=-1).float()
    topk_ids = torch.zeros(M, top_k, dtype=torch.int32)
    for i in range(M):
        topk_ids[i] = torch.randperm(E, generator=torch.Generator().manual_seed(i))[
            :top_k
        ].int()

    output = torch.empty(M, K, dtype=torch.bfloat16)
    ops.fused_experts_cpu(
        output,
        x_fp8,
        packed_w13,
        packed_w2,
        topk_weights,
        topk_ids,
        ops.CPUQuantMethod.FP8_W8A8,
        packed_w13_scale,  # w1_scale: [E, Nc, G, BLOCK_N]
        w2_scale,  # w2_scale: [E, K//128, N//128]
        None,  # w1_zero
        None,  # w2_zero
        [FP8_W8A8_QUANT_GROUP, FP8_W8A8_QUANT_GROUP],  # block_size
        None,  # w1_bias
        None,  # w2_bias
        None,  # alpha
        None,  # limit
        True,  # is_vnni (weights already packed)
        x_scales,  # a1_scale: per-token FP8 activation scales
    )

    assert output.shape == (M, K)
    assert output.dtype == torch.bfloat16
    assert not torch.isnan(output).any()
    assert not torch.isinf(output).any()

    x_dequant = (x_fp8.float() * x_scales.unsqueeze(1)).bfloat16()
    ref = _ref_fp8_w8a8_moe(
        x_dequant, w13, w13_scale, w2, w2_scale, topk_weights, topk_ids.long(), N
    )
    torch.testing.assert_close(output.float(), ref.float(), atol=1e-2, rtol=1e-2)


if __name__ == "__main__":
    sys.exit(pytest.main([__file__, "-v"]))