[CPU] Support cpu compressed-tensor w8a8 int8 moe (#42920)

Signed-off-by: yuwenzho <yuwen.zhou@intel.com>
Signed-off-by: Yuwen Zhou <yuwen.zhou@intel.com>
This commit is contained in:
Yuwen Zhou
2026-06-29 03:04:05 +00:00
committed by GitHub
parent a2abce646f
commit 58d6a6e60a
6 changed files with 410 additions and 9 deletions
+139 -4
View File
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for CPU quantized fused MoE kernels (FP8 W8A16 and MXFP4 W4A16)."""
"""Tests for CPU quantized fused MoE kernels."""
import math
import sys
@@ -31,7 +31,10 @@ def _prepack_experts(w: torch.Tensor) -> torch.Tensor:
return torch.ops._C.convert_weight_packed(w)
# FP8 W8A16 block-scaled fused MoE
# ===========================================================================
# FP8 W8A16 MoE
# ===========================================================================
BLOCK_SIZE = [128, 128] # [block_n, block_k]
@@ -216,7 +219,9 @@ def test_w8a16_block_fp8_cpu_fused_moe(M, N, K, E, topk, seed):
torch.testing.assert_close(out_inplace, out, atol=0, rtol=0)
# MXFP4 W4A16 fused MoE
# ===========================================================================
# MXFP4 W4A16 MoE
# ===========================================================================
class MXFP4QuantizeUtil:
@@ -496,7 +501,9 @@ def test_mxfp4_cpu_fused_moe_bias_swiglu(M, N, K, E, topk, seed):
torch.testing.assert_close(ref_out.bfloat16(), out, atol=1e-2, rtol=1e-2)
# INT4 W4A16 group-quantized MoE
# ===========================================================================
# INT4 W4A16 MoE
# ===========================================================================
def _pack_int4_gptq(w_int4: torch.Tensor) -> torch.Tensor:
@@ -749,5 +756,133 @@ def test_int4_w4a16_cpu_fused_moe(M, N, K, E, topk, group_size, quant_algo, seed
torch.testing.assert_close(ref_out.bfloat16(), out, atol=1e-2, rtol=1e-2)
# ===========================================================================
# INT8 W8A8 MoE
# ===========================================================================
def _quantize_per_channel(w):
"""Symmetric per-channel INT8 quantisation. w: [N, K] -> (int8, scale)."""
amax = w.abs().amax(dim=-1, keepdim=True).clamp(min=1e-12)
scale = amax / 127.0
w_q = (w / scale).round().clamp(-128, 127).to(torch.int8)
return w_q, scale.float()
def _quantize_per_token(x):
"""Symmetric per-token INT8 quantisation. x: [M, K] -> (int8, scale)."""
amax = x.abs().amax(dim=-1, keepdim=True).clamp(min=1e-12)
scale = amax / 127.0
x_q = (x / scale).round().clamp(-128, 127).to(torch.int8)
return x_q, scale.float()
def _ref_int8_moe(a, w1, w2, w1_s, w2_s, topk_weight, topk_ids):
"""Reference INT8 W8A8 per-channel fused MoE in pure torch."""
B, D = a.shape
topk = topk_ids.size(1)
out = torch.zeros(B, topk, w2.shape[1], dtype=torch.float32)
for b in range(B):
for t in range(topk):
eid = topk_ids[b, t].item()
x = a[b : b + 1].float()
x_q, x_s = _quantize_per_token(x)
ic = torch.matmul(x_q.float(), w1[eid].float().t())
ic = ic * x_s * w1_s[eid].view(1, -1)
ic = _silu_and_mul(ic)
ic_q, ic_s = _quantize_per_token(ic)
oc = torch.matmul(ic_q.float(), w2[eid].float().t())
oc = oc * ic_s * w2_s[eid].view(1, -1)
out[b, t] = oc.squeeze(0)
result = (out * topk_weight.unsqueeze(-1)).sum(dim=1)
return result.to(a.dtype)
def _make_int8_moe_weights(E, N, K):
factor = 1e-2
w1_f = (torch.randn(E, 2 * N, K) - 0.5) * 2
w2_f = (torch.randn(E, K, N) - 0.5) * 2
w1_q_list, w1_s_list = [], []
w2_q_list, w2_s_list = [], []
for e in range(E):
q, s = _quantize_per_channel(w1_f[e])
w1_q_list.append(q)
w1_s_list.append(s)
q, s = _quantize_per_channel(w2_f[e])
w2_q_list.append(q)
w2_s_list.append(s)
return (
torch.stack(w1_q_list),
torch.stack(w2_q_list),
torch.stack(w1_s_list) * factor,
torch.stack(w2_s_list) * factor,
)
INT8_NUM_TOKENS = [1, 2, 64, 121]
INT8_MOE_CONFIGS = [
# (N, K, E, topk)
(256, 512, 8, 2),
(512, 256, 8, 2),
(512, 512, 8, 4),
(768, 2048, 8, 2),
]
@pytest.mark.parametrize("M", INT8_NUM_TOKENS)
@pytest.mark.parametrize("N,K,E,topk", INT8_MOE_CONFIGS)
@pytest.mark.parametrize("seed", [0])
@pytest.mark.parametrize("is_vnni", [False, True])
@pytest.mark.parametrize("inplace", [False, True])
def test_int8_w8a8_cpu_fused_moe(M, N, K, E, topk, seed, is_vnni, inplace):
"""Test fused_experts_cpu INT8 W8A8 against torch reference."""
set_random_seed(seed)
a = torch.randn(M, K, dtype=torch.bfloat16) / (0.5 * K**0.5)
w1_q, w2_q, w1_s, w2_s = _make_int8_moe_weights(E, N, K)
score = torch.randn(M, E, dtype=torch.bfloat16)
score = torch.softmax(score, dim=-1, dtype=torch.float32)
topk_weight, topk_ids = torch.topk(score, topk)
topk_ids = topk_ids.to(torch.int32)
ref_out = _ref_int8_moe(a, w1_q, w2_q, w1_s, w2_s, topk_weight, topk_ids)
w1 = _prepack_experts(w1_q) if is_vnni else w1_q
w2 = _prepack_experts(w2_q) if is_vnni else w2_q
out = ops.fused_experts_cpu(
a.clone(),
w1,
w2,
topk_weight,
topk_ids,
inplace,
ops.CPUQuantMethod.INT8_W8A8,
w1_s,
w2_s,
None, # w1_zero
None, # w2_zero
None, # block_size
None, # w1_bias
None, # w2_bias
None, # alpha
None, # limit
is_vnni,
)
torch.testing.assert_close(
ref_out.bfloat16(),
out,
atol=2e-1,
rtol=2e-1,
)
if __name__ == "__main__":
sys.exit(pytest.main([__file__, "-v"]))
+22
View File
@@ -0,0 +1,22 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from vllm.platforms import current_platform
if not current_platform.is_cpu():
pytest.skip("skipping CPU-only tests", allow_module_level=True)
MODELS = [
"RedHatAI/Qwen3-30B-A3B-Instruct-2507-quantized.w8a8", # INT8 W8A8 MoE
]
DTYPE = ["bfloat16"]
@pytest.mark.parametrize("model", MODELS)
@pytest.mark.parametrize("dtype", DTYPE)
def test_cpu_w8a8(vllm_runner, model, dtype):
with vllm_runner(model, dtype=dtype) as llm:
output = llm.generate_greedy(["The capital of France is"], max_tokens=32)
assert output
print(output)