[CPU] Support cpu compressed-tensor w8a8 int8 moe (#42920)
Signed-off-by: yuwenzho <yuwen.zhou@intel.com> Signed-off-by: Yuwen Zhou <yuwen.zhou@intel.com>
This commit is contained in:
@@ -1,6 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for CPU quantized fused MoE kernels (FP8 W8A16 and MXFP4 W4A16)."""
|
||||
"""Tests for CPU quantized fused MoE kernels."""
|
||||
|
||||
import math
|
||||
import sys
|
||||
@@ -31,7 +31,10 @@ def _prepack_experts(w: torch.Tensor) -> torch.Tensor:
|
||||
return torch.ops._C.convert_weight_packed(w)
|
||||
|
||||
|
||||
# FP8 W8A16 block-scaled fused MoE
|
||||
# ===========================================================================
|
||||
# FP8 W8A16 MoE
|
||||
# ===========================================================================
|
||||
|
||||
|
||||
BLOCK_SIZE = [128, 128] # [block_n, block_k]
|
||||
|
||||
@@ -216,7 +219,9 @@ def test_w8a16_block_fp8_cpu_fused_moe(M, N, K, E, topk, seed):
|
||||
torch.testing.assert_close(out_inplace, out, atol=0, rtol=0)
|
||||
|
||||
|
||||
# MXFP4 W4A16 fused MoE
|
||||
# ===========================================================================
|
||||
# MXFP4 W4A16 MoE
|
||||
# ===========================================================================
|
||||
|
||||
|
||||
class MXFP4QuantizeUtil:
|
||||
@@ -496,7 +501,9 @@ def test_mxfp4_cpu_fused_moe_bias_swiglu(M, N, K, E, topk, seed):
|
||||
torch.testing.assert_close(ref_out.bfloat16(), out, atol=1e-2, rtol=1e-2)
|
||||
|
||||
|
||||
# INT4 W4A16 group-quantized MoE
|
||||
# ===========================================================================
|
||||
# INT4 W4A16 MoE
|
||||
# ===========================================================================
|
||||
|
||||
|
||||
def _pack_int4_gptq(w_int4: torch.Tensor) -> torch.Tensor:
|
||||
@@ -749,5 +756,133 @@ def test_int4_w4a16_cpu_fused_moe(M, N, K, E, topk, group_size, quant_algo, seed
|
||||
torch.testing.assert_close(ref_out.bfloat16(), out, atol=1e-2, rtol=1e-2)
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# INT8 W8A8 MoE
|
||||
# ===========================================================================
|
||||
|
||||
|
||||
def _quantize_per_channel(w):
|
||||
"""Symmetric per-channel INT8 quantisation. w: [N, K] -> (int8, scale)."""
|
||||
amax = w.abs().amax(dim=-1, keepdim=True).clamp(min=1e-12)
|
||||
scale = amax / 127.0
|
||||
w_q = (w / scale).round().clamp(-128, 127).to(torch.int8)
|
||||
return w_q, scale.float()
|
||||
|
||||
|
||||
def _quantize_per_token(x):
|
||||
"""Symmetric per-token INT8 quantisation. x: [M, K] -> (int8, scale)."""
|
||||
amax = x.abs().amax(dim=-1, keepdim=True).clamp(min=1e-12)
|
||||
scale = amax / 127.0
|
||||
x_q = (x / scale).round().clamp(-128, 127).to(torch.int8)
|
||||
return x_q, scale.float()
|
||||
|
||||
|
||||
def _ref_int8_moe(a, w1, w2, w1_s, w2_s, topk_weight, topk_ids):
|
||||
"""Reference INT8 W8A8 per-channel fused MoE in pure torch."""
|
||||
B, D = a.shape
|
||||
topk = topk_ids.size(1)
|
||||
|
||||
out = torch.zeros(B, topk, w2.shape[1], dtype=torch.float32)
|
||||
for b in range(B):
|
||||
for t in range(topk):
|
||||
eid = topk_ids[b, t].item()
|
||||
|
||||
x = a[b : b + 1].float()
|
||||
x_q, x_s = _quantize_per_token(x)
|
||||
ic = torch.matmul(x_q.float(), w1[eid].float().t())
|
||||
ic = ic * x_s * w1_s[eid].view(1, -1)
|
||||
ic = _silu_and_mul(ic)
|
||||
|
||||
ic_q, ic_s = _quantize_per_token(ic)
|
||||
oc = torch.matmul(ic_q.float(), w2[eid].float().t())
|
||||
oc = oc * ic_s * w2_s[eid].view(1, -1)
|
||||
out[b, t] = oc.squeeze(0)
|
||||
|
||||
result = (out * topk_weight.unsqueeze(-1)).sum(dim=1)
|
||||
return result.to(a.dtype)
|
||||
|
||||
|
||||
def _make_int8_moe_weights(E, N, K):
|
||||
factor = 1e-2
|
||||
w1_f = (torch.randn(E, 2 * N, K) - 0.5) * 2
|
||||
w2_f = (torch.randn(E, K, N) - 0.5) * 2
|
||||
|
||||
w1_q_list, w1_s_list = [], []
|
||||
w2_q_list, w2_s_list = [], []
|
||||
for e in range(E):
|
||||
q, s = _quantize_per_channel(w1_f[e])
|
||||
w1_q_list.append(q)
|
||||
w1_s_list.append(s)
|
||||
q, s = _quantize_per_channel(w2_f[e])
|
||||
w2_q_list.append(q)
|
||||
w2_s_list.append(s)
|
||||
|
||||
return (
|
||||
torch.stack(w1_q_list),
|
||||
torch.stack(w2_q_list),
|
||||
torch.stack(w1_s_list) * factor,
|
||||
torch.stack(w2_s_list) * factor,
|
||||
)
|
||||
|
||||
|
||||
INT8_NUM_TOKENS = [1, 2, 64, 121]
|
||||
INT8_MOE_CONFIGS = [
|
||||
# (N, K, E, topk)
|
||||
(256, 512, 8, 2),
|
||||
(512, 256, 8, 2),
|
||||
(512, 512, 8, 4),
|
||||
(768, 2048, 8, 2),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("M", INT8_NUM_TOKENS)
|
||||
@pytest.mark.parametrize("N,K,E,topk", INT8_MOE_CONFIGS)
|
||||
@pytest.mark.parametrize("seed", [0])
|
||||
@pytest.mark.parametrize("is_vnni", [False, True])
|
||||
@pytest.mark.parametrize("inplace", [False, True])
|
||||
def test_int8_w8a8_cpu_fused_moe(M, N, K, E, topk, seed, is_vnni, inplace):
|
||||
"""Test fused_experts_cpu INT8 W8A8 against torch reference."""
|
||||
set_random_seed(seed)
|
||||
|
||||
a = torch.randn(M, K, dtype=torch.bfloat16) / (0.5 * K**0.5)
|
||||
w1_q, w2_q, w1_s, w2_s = _make_int8_moe_weights(E, N, K)
|
||||
|
||||
score = torch.randn(M, E, dtype=torch.bfloat16)
|
||||
score = torch.softmax(score, dim=-1, dtype=torch.float32)
|
||||
topk_weight, topk_ids = torch.topk(score, topk)
|
||||
topk_ids = topk_ids.to(torch.int32)
|
||||
|
||||
ref_out = _ref_int8_moe(a, w1_q, w2_q, w1_s, w2_s, topk_weight, topk_ids)
|
||||
|
||||
w1 = _prepack_experts(w1_q) if is_vnni else w1_q
|
||||
w2 = _prepack_experts(w2_q) if is_vnni else w2_q
|
||||
|
||||
out = ops.fused_experts_cpu(
|
||||
a.clone(),
|
||||
w1,
|
||||
w2,
|
||||
topk_weight,
|
||||
topk_ids,
|
||||
inplace,
|
||||
ops.CPUQuantMethod.INT8_W8A8,
|
||||
w1_s,
|
||||
w2_s,
|
||||
None, # w1_zero
|
||||
None, # w2_zero
|
||||
None, # block_size
|
||||
None, # w1_bias
|
||||
None, # w2_bias
|
||||
None, # alpha
|
||||
None, # limit
|
||||
is_vnni,
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
ref_out.bfloat16(),
|
||||
out,
|
||||
atol=2e-1,
|
||||
rtol=2e-1,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(pytest.main([__file__, "-v"]))
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import pytest
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if not current_platform.is_cpu():
|
||||
pytest.skip("skipping CPU-only tests", allow_module_level=True)
|
||||
|
||||
MODELS = [
|
||||
"RedHatAI/Qwen3-30B-A3B-Instruct-2507-quantized.w8a8", # INT8 W8A8 MoE
|
||||
]
|
||||
DTYPE = ["bfloat16"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", MODELS)
|
||||
@pytest.mark.parametrize("dtype", DTYPE)
|
||||
def test_cpu_w8a8(vllm_runner, model, dtype):
|
||||
with vllm_runner(model, dtype=dtype) as llm:
|
||||
output = llm.generate_greedy(["The capital of France is"], max_tokens=32)
|
||||
assert output
|
||||
print(output)
|
||||
Reference in New Issue
Block a user