[CPU][BugFix] Multiple fixes to w4a8_int8 CPU MoE path (#46739)
Signed-off-by: Fadi Arafeh <fadi.arafeh@arm.com>
This commit is contained in:
@@ -38,7 +38,8 @@ function cpu_tests() {
|
||||
pytest -x -v -s tests/kernels/attention/test_cpu_attn.py
|
||||
pytest -x -v -s tests/kernels/core/test_cpu_activation.py
|
||||
pytest -x -v -s tests/kernels/moe/test_cpu_fused_moe.py
|
||||
pytest -x -v -s tests/kernels/mamba/cpu/test_cpu_gdn_ops.py"
|
||||
pytest -x -v -s tests/kernels/mamba/cpu/test_cpu_gdn_ops.py
|
||||
pytest -x -v -s tests/kernels/moe/test_cpu_int4_moe.py"
|
||||
|
||||
# skip tests requiring model downloads if HF_TOKEN is not set
|
||||
# due to rate-limits
|
||||
|
||||
@@ -278,7 +278,8 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
|
||||
ops.def(
|
||||
"dynamic_4bit_int_moe("
|
||||
"Tensor x, Tensor topk_ids, Tensor topk_weights,"
|
||||
"Tensor w13_packed, Tensor w2_packed, int H, int I, int I2,"
|
||||
"Tensor w13_packed, Tensor w2_packed,"
|
||||
"int hidden_size, int intermediate_size,"
|
||||
"int group_size, bool apply_router_weight_on_input, int activation_kind"
|
||||
") -> Tensor");
|
||||
|
||||
|
||||
@@ -29,25 +29,37 @@ enum ActivationKind : int64_t {
|
||||
|
||||
torch::Tensor dynamic_4bit_int_moe_cpu(
|
||||
torch::Tensor x, torch::Tensor topk_ids, torch::Tensor topk_weights,
|
||||
torch::Tensor w13_packed, torch::Tensor w2_packed, int64_t H, int64_t I,
|
||||
int64_t I2, int64_t group_size, bool apply_router_weight_on_input,
|
||||
int64_t activation_kind) {
|
||||
torch::Tensor w13_packed, torch::Tensor w2_packed, int64_t hidden_size,
|
||||
int64_t intermediate_size, int64_t group_size,
|
||||
bool apply_router_weight_on_input, int64_t activation_kind) {
|
||||
TORCH_CHECK(x.dim() == 2, "x must be 2D");
|
||||
TORCH_CHECK(topk_ids.dim() == 2 && topk_weights.dim() == 2,
|
||||
"topk tensors must be [T, K]");
|
||||
TORCH_CHECK(
|
||||
w13_packed.size(0) == w2_packed.size(0),
|
||||
"w13_packed and w2_packed must have same number of experts in dim 0");
|
||||
TORCH_CHECK(I2 == 2 * I, "I2 must equal 2*I");
|
||||
|
||||
const int64_t T = x.size(0);
|
||||
const int64_t K = topk_ids.size(1);
|
||||
const int64_t E = w13_packed.size(0);
|
||||
const int64_t N = T * K;
|
||||
const int64_t w13_out_features = 2 * intermediate_size;
|
||||
|
||||
auto x_c = x.contiguous();
|
||||
// _dyn_quant_matmul_4bit kernel natively supports these pre-quant activation
|
||||
// dtypes:
|
||||
// - fp32: with channelwise and groupwise
|
||||
// - bf16: with channelwise -> upcast to fp32 for groupwise
|
||||
// - fp16: not supported -> upcast to fp32 for groupwise & channelwise
|
||||
const auto output_dtype = x_c.scalar_type();
|
||||
const bool should_cast_input =
|
||||
((group_size != -1) && output_dtype == at::kBFloat16) ||
|
||||
output_dtype == at::kHalf;
|
||||
if (should_cast_input) {
|
||||
x_c = x_c.to(at::kFloat);
|
||||
}
|
||||
auto ids_c = topk_ids.contiguous();
|
||||
auto gates_c = topk_weights.to(at::kFloat).contiguous();
|
||||
auto gates_c = topk_weights.to(x_c.scalar_type()).contiguous();
|
||||
|
||||
// bucketing tokens -> experts
|
||||
c10::SmallVector<int64_t, 64> counts(
|
||||
@@ -63,35 +75,42 @@ torch::Tensor dynamic_4bit_int_moe_cpu(
|
||||
c10::SmallVector<int64_t, 65> offsets(E + 1, 0); // ( E +1 )
|
||||
for (int64_t e = 0; e < E; ++e) offsets[e + 1] = offsets[e] + counts[e];
|
||||
|
||||
// expert_tokens = [tokens indices for expert 0, ...]
|
||||
// expert_gates = [router weights for tokens assigned to expert 0, ...]
|
||||
auto expert_tokens = at::empty({offsets[E]}, ids_c.options());
|
||||
auto expert_gates = at::empty({offsets[E]}, gates_c.options());
|
||||
{
|
||||
c10::SmallVector<int64_t, 64> cursor(E, 0);
|
||||
const auto* ids_ptr = ids_c.data_ptr<int64_t>();
|
||||
const auto* gts_ptr = gates_c.data_ptr<float>();
|
||||
auto* tok_ptr = expert_tokens.data_ptr<int64_t>();
|
||||
auto* gate_ptr = expert_gates.data_ptr<float>();
|
||||
AT_DISPATCH_FLOATING_TYPES_AND2(
|
||||
at::ScalarType::BFloat16, at::ScalarType::Half, gates_c.scalar_type(),
|
||||
"bucket_expert_tokens_and_gates", [&] {
|
||||
const auto* ids_ptr = ids_c.data_ptr<int64_t>();
|
||||
const auto* gts_ptr = gates_c.data_ptr<scalar_t>();
|
||||
auto* tok_ptr = expert_tokens.data_ptr<int64_t>();
|
||||
auto* gate_ptr = expert_gates.data_ptr<scalar_t>();
|
||||
|
||||
for (int64_t t = 0; t < T; ++t) {
|
||||
const int64_t base = t * K;
|
||||
for (int64_t k = 0; k < K; ++k) {
|
||||
const int64_t idx = base + k;
|
||||
const int64_t e = ids_ptr[idx];
|
||||
const int64_t p = offsets[e] + (cursor[e]++);
|
||||
tok_ptr[p] = t;
|
||||
gate_ptr[p] = gts_ptr[idx];
|
||||
}
|
||||
}
|
||||
for (int64_t t = 0; t < T; ++t) {
|
||||
const int64_t base = t * K;
|
||||
for (int64_t k = 0; k < K; ++k) {
|
||||
const int64_t idx = base + k;
|
||||
const int64_t e = ids_ptr[idx];
|
||||
const int64_t p = offsets[e] + (cursor[e]++);
|
||||
tok_ptr[p] = t;
|
||||
gate_ptr[p] = gts_ptr[idx];
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
const int64_t g_eff_13 = (group_size != -1) ? group_size : H;
|
||||
const int64_t g_eff_2 = (group_size != -1) ? group_size : I;
|
||||
const int64_t g_eff_13 = (group_size != -1) ? group_size : hidden_size;
|
||||
const int64_t g_eff_2 = (group_size != -1) ? group_size : intermediate_size;
|
||||
|
||||
// X_all [num_tokens * K, hidden_size]
|
||||
auto X_all = x_c.index_select(/*dim=*/0, expert_tokens);
|
||||
if (apply_router_weight_on_input) {
|
||||
X_all = X_all.mul(expert_gates.unsqueeze(1));
|
||||
}
|
||||
auto Y_all = at::empty({offsets[E], H}, x_c.options());
|
||||
auto Y_all = at::empty({offsets[E], hidden_size}, x_c.options());
|
||||
|
||||
at::parallel_for(0, offsets[E], 0, [&](int64_t idx_begin, int64_t idx_end) {
|
||||
c10::InferenceMode guard;
|
||||
@@ -109,11 +128,13 @@ torch::Tensor dynamic_4bit_int_moe_cpu(
|
||||
auto w2_e = w2_packed.select(/*dim=*/0, e);
|
||||
|
||||
// W13
|
||||
auto y13 =
|
||||
mm(x_e, w13_e, g_eff_13, /*in_features=*/H, /*out_features=*/I2);
|
||||
auto y13 = mm(x_e, w13_e, g_eff_13, /*in_features=*/hidden_size,
|
||||
/*out_features=*/w13_out_features);
|
||||
|
||||
auto g_part = y13.narrow(/*dim=*/1, /*start=*/0, /*length=*/I);
|
||||
auto u_part = y13.narrow(/*dim=*/1, /*start=*/I, /*length=*/I);
|
||||
auto g_part =
|
||||
y13.narrow(/*dim=*/1, /*start=*/0, /*length=*/intermediate_size);
|
||||
auto u_part = y13.narrow(/*dim=*/1, /*start=*/intermediate_size,
|
||||
/*length=*/intermediate_size);
|
||||
|
||||
torch::Tensor act;
|
||||
if (activation_kind == ActivationKind::SwiGLUOAI) { // SwiGLUOAI
|
||||
@@ -128,7 +149,8 @@ torch::Tensor dynamic_4bit_int_moe_cpu(
|
||||
}
|
||||
|
||||
// W2
|
||||
auto y = mm(act, w2_e, g_eff_2, /*in_features=*/I, /*out_features=*/H);
|
||||
auto y = mm(act, w2_e, g_eff_2, /*in_features=*/intermediate_size,
|
||||
/*out_features=*/hidden_size);
|
||||
|
||||
// Store per-expert result
|
||||
Y_all.narrow(/*dim=*/0, /*start=*/start, /*length=*/te).copy_(y);
|
||||
@@ -138,8 +160,11 @@ torch::Tensor dynamic_4bit_int_moe_cpu(
|
||||
if (!apply_router_weight_on_input) {
|
||||
Y_all = Y_all.mul(expert_gates.unsqueeze(1));
|
||||
}
|
||||
if (Y_all.scalar_type() != output_dtype) {
|
||||
Y_all = Y_all.to(output_dtype);
|
||||
}
|
||||
|
||||
auto out = at::zeros({T, H}, x.options());
|
||||
auto out = at::zeros({T, hidden_size}, x.options());
|
||||
out =
|
||||
at::index_add(out, /*dim=*/0, /*index=*/expert_tokens, /*source=*/Y_all);
|
||||
|
||||
|
||||
+3
-3
@@ -53,9 +53,9 @@ void dynamic_scaled_int8_quant(torch::Tensor& out, torch::Tensor const& input,
|
||||
|
||||
torch::Tensor dynamic_4bit_int_moe_cpu(
|
||||
torch::Tensor x, torch::Tensor topk_ids, torch::Tensor topk_weights,
|
||||
torch::Tensor w13_packed, torch::Tensor w2_packed, int64_t H, int64_t I,
|
||||
int64_t I2, int64_t group_size, bool apply_router_weight_on_input,
|
||||
int64_t activation_kind);
|
||||
torch::Tensor w13_packed, torch::Tensor w2_packed, int64_t hidden_size,
|
||||
int64_t intermediate_size, int64_t group_size,
|
||||
bool apply_router_weight_on_input, int64_t activation_kind);
|
||||
|
||||
using fptr_t = int64_t;
|
||||
#ifdef USE_ROCM
|
||||
|
||||
@@ -8,19 +8,21 @@ import pytest
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.experts.cpu_int4_moe import (
|
||||
CPUExpertsInt4,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.oracle.w4a8_int8 import (
|
||||
convert_to_w4a8_int8_moe_format,
|
||||
)
|
||||
from vllm.platforms import CpuArchEnum, current_platform
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
if not current_platform.is_cpu():
|
||||
pytest.skip("skipping CPU-only tests", allow_module_level=True)
|
||||
|
||||
# Check if the dynamic_4bit_int_moe op is available
|
||||
if not hasattr(torch.ops._C, "dynamic_4bit_int_moe"):
|
||||
pytest.skip("dynamic_4bit_int_moe op not available", allow_module_level=True)
|
||||
|
||||
# Check if KleidiAI ops are available
|
||||
if not hasattr(torch.ops.aten, "_dyn_quant_pack_4bit_weight"):
|
||||
pytest.skip("KleidiAI 4-bit ops not available", allow_module_level=True)
|
||||
if (
|
||||
not current_platform.is_cpu()
|
||||
or current_platform.get_cpu_architecture() != CpuArchEnum.ARM
|
||||
):
|
||||
pytest.skip("skipping Arm CPU-only tests", allow_module_level=True)
|
||||
|
||||
|
||||
# Tolerance for INT4 W4A8
|
||||
@@ -34,49 +36,6 @@ def _silu_and_mul(x: torch.Tensor) -> torch.Tensor:
|
||||
return F.silu(x[..., :d]) * x[..., d:]
|
||||
|
||||
|
||||
def _pack_int4_weight_to_kleidi(
|
||||
int4_as_int8: torch.Tensor,
|
||||
scales: torch.Tensor,
|
||||
bias: torch.Tensor | None,
|
||||
group_size: int,
|
||||
in_features: int,
|
||||
out_features: int,
|
||||
) -> torch.Tensor:
|
||||
"""Pack INT4 weights (stored as int8 in [-8,7]) to KleidiAI format.
|
||||
|
||||
Args:
|
||||
int4_as_int8: [out, in] int8 tensor with values in [-8, 7]
|
||||
scales: [out, in//group_size] or [out, 1] for channel-wise
|
||||
bias: [out] optional bias
|
||||
group_size: Quantization group size (-1 for channel-wise)
|
||||
in_features: Input dimension
|
||||
out_features: Output dimension
|
||||
|
||||
Returns:
|
||||
Packed weight tensor in KleidiAI format
|
||||
"""
|
||||
# Shift to unsigned nibble [0, 15]
|
||||
tmp = int4_as_int8.add(8)
|
||||
# Pack pairs along input dimension
|
||||
uint8_nibbles = ((tmp[:, 1::2] << 4) | tmp[:, ::2]).to(torch.uint8)
|
||||
|
||||
# Determine scale dtype based on group_size
|
||||
scale_dtype = torch.float32 if group_size == -1 else torch.bfloat16
|
||||
scales_typed = scales.to(scale_dtype)
|
||||
bias_typed = None if bias is None else bias.to(torch.float32)
|
||||
|
||||
# Pack using KleidiAI op
|
||||
actual_group_size = in_features if group_size == -1 else group_size
|
||||
return torch.ops.aten._dyn_quant_pack_4bit_weight(
|
||||
uint8_nibbles,
|
||||
scales_typed,
|
||||
bias_typed,
|
||||
actual_group_size,
|
||||
in_features,
|
||||
out_features,
|
||||
)
|
||||
|
||||
|
||||
def _make_int4_moe_weights(
|
||||
E: int,
|
||||
N: int,
|
||||
@@ -124,59 +83,29 @@ def _make_int4_moe_weights(
|
||||
w13_bias = torch.randn(E, 2 * N, dtype=torch.float32) * 0.01
|
||||
w2_bias = torch.randn(E, K, dtype=torch.float32) * 0.01
|
||||
|
||||
# Pack weights for each expert
|
||||
w13_packed_list = []
|
||||
w2_packed_list = []
|
||||
w13_packed, w2_packed, *_ = convert_to_w4a8_int8_moe_format(
|
||||
w13_weight=w13_int4,
|
||||
w2_weight=w2_int4,
|
||||
w13_weight_scale=w13_scales,
|
||||
w2_weight_scale=w2_scales,
|
||||
group_size=group_size,
|
||||
w13_bias=w13_bias if has_bias else None,
|
||||
w2_bias=w2_bias if has_bias else None,
|
||||
)
|
||||
|
||||
for e in range(E):
|
||||
w13_packed_list.append(
|
||||
_pack_int4_weight_to_kleidi(
|
||||
w13_int4[e],
|
||||
w13_scales[e],
|
||||
w13_bias[e] if (has_bias and w13_bias is not None) else None,
|
||||
group_size,
|
||||
K,
|
||||
2 * N,
|
||||
)
|
||||
)
|
||||
w2_packed_list.append(
|
||||
_pack_int4_weight_to_kleidi(
|
||||
w2_int4[e],
|
||||
w2_scales[e],
|
||||
w2_bias[e] if (has_bias and w2_bias is not None) else None,
|
||||
group_size,
|
||||
N,
|
||||
K,
|
||||
)
|
||||
)
|
||||
if group_size == -1:
|
||||
w13_scale = w13_scales.float()
|
||||
w2_scale = w2_scales.float()
|
||||
else:
|
||||
w13_scale = w13_scales.float().repeat_interleave(group_size, dim=-1)
|
||||
w2_scale = w2_scales.float().repeat_interleave(group_size, dim=-1)
|
||||
|
||||
w13_packed = torch.stack(w13_packed_list, dim=0)
|
||||
w2_packed = torch.stack(w2_packed_list, dim=0)
|
||||
|
||||
# Create reference dequantized weights
|
||||
w13_ref = torch.zeros(E, 2 * N, K, dtype=torch.float32)
|
||||
w2_ref = torch.zeros(E, K, N, dtype=torch.float32)
|
||||
|
||||
for e in range(E):
|
||||
# Dequantize w13
|
||||
for i in range(2 * N):
|
||||
for j in range(K):
|
||||
group_idx = 0 if group_size == -1 else (j // group_size)
|
||||
w13_ref[e, i, j] = (
|
||||
w13_int4[e, i, j].float() * w13_scales[e, i, group_idx].float()
|
||||
)
|
||||
if has_bias and w13_bias is not None:
|
||||
w13_ref[e, i, j] += w13_bias[e, i].float()
|
||||
|
||||
# Dequantize w2
|
||||
for i in range(K):
|
||||
for j in range(N):
|
||||
group_idx = 0 if group_size == -1 else (j // group_size)
|
||||
w2_ref[e, i, j] = (
|
||||
w2_int4[e, i, j].float() * w2_scales[e, i, group_idx].float()
|
||||
)
|
||||
if has_bias and w2_bias is not None:
|
||||
w2_ref[e, i, j] += w2_bias[e, i].float()
|
||||
w13_ref = w13_int4.float() * w13_scale
|
||||
w2_ref = w2_int4.float() * w2_scale
|
||||
if has_bias and w13_bias is not None:
|
||||
w13_ref = w13_ref + w13_bias.float().unsqueeze(-1)
|
||||
if has_bias and w2_bias is not None:
|
||||
w2_ref = w2_ref + w2_bias.float().unsqueeze(-1)
|
||||
|
||||
return w13_packed, w2_packed, w13_ref, w2_ref, w13_bias, w2_bias
|
||||
|
||||
@@ -233,17 +162,20 @@ MoE_CONFIGS = [
|
||||
(768, 2048, 16, 4, 64),
|
||||
]
|
||||
SEEDS = [0, 42]
|
||||
ACTIVATION_DTYPES = [torch.float32, torch.bfloat16, torch.float16]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("M", NUM_TOKENS)
|
||||
@pytest.mark.parametrize("N,K,E,topk,group_size", MoE_CONFIGS)
|
||||
@pytest.mark.parametrize("seed", SEEDS)
|
||||
def test_cpu_int4_moe_kernel(M, N, K, E, topk, group_size, seed):
|
||||
@pytest.mark.parametrize("activation_dtype", ACTIVATION_DTYPES)
|
||||
def test_cpu_int4_moe_kernel(M, N, K, E, topk, group_size, seed, activation_dtype):
|
||||
"""Test dynamic_4bit_int_moe kernel against dequantized torch reference."""
|
||||
set_random_seed(seed)
|
||||
activation = MoEActivation.SILU
|
||||
|
||||
# Generate input activations
|
||||
a = torch.randn(M, K, dtype=torch.bfloat16) / (K**0.5)
|
||||
a = torch.randn(M, K, dtype=activation_dtype) / (K**0.5)
|
||||
|
||||
# Generate INT4 weights
|
||||
w13_packed, w2_packed, w13_ref, w2_ref, w13_bias, w2_bias = _make_int4_moe_weights(
|
||||
@@ -266,8 +198,6 @@ def test_cpu_int4_moe_kernel(M, N, K, E, topk, group_size, seed):
|
||||
)
|
||||
|
||||
# Test dynamic_4bit_int_moe kernel
|
||||
# Activation kind: 1 = SwiGLU_Ug (SiLU(u)*g) for OAI-style
|
||||
activation_kind = 1
|
||||
apply_router_weight_on_input = False
|
||||
|
||||
out = torch.ops._C.dynamic_4bit_int_moe(
|
||||
@@ -278,14 +208,14 @@ def test_cpu_int4_moe_kernel(M, N, K, E, topk, group_size, seed):
|
||||
w2_packed,
|
||||
K, # H (hidden_size / w2_out_features)
|
||||
N, # I (intermediate_size / w2_in_features)
|
||||
2 * N, # I2 (2*intermediate_size / w13_out_features)
|
||||
group_size,
|
||||
apply_router_weight_on_input,
|
||||
activation_kind,
|
||||
CPUExpertsInt4._activation_kind(activation),
|
||||
)
|
||||
|
||||
assert out.dtype == activation_dtype
|
||||
torch.testing.assert_close(
|
||||
ref_out.bfloat16(),
|
||||
ref_out,
|
||||
out,
|
||||
atol=INT4_W4A8_ATOL,
|
||||
rtol=INT4_W4A8_RTOL,
|
||||
|
||||
@@ -2974,6 +2974,24 @@ if hasattr(torch.ops._C, "fused_experts_cpu"):
|
||||
return torch.empty_like(hidden_states)
|
||||
|
||||
|
||||
if hasattr(torch.ops._C, "dynamic_4bit_int_moe"):
|
||||
|
||||
@register_fake("_C::dynamic_4bit_int_moe")
|
||||
def dynamic_4bit_int_moe_fake(
|
||||
x: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
w13_packed: torch.Tensor,
|
||||
w2_packed: torch.Tensor,
|
||||
hidden_size: int,
|
||||
intermediate_size: int,
|
||||
group_size: int,
|
||||
apply_router_weight_on_input: bool,
|
||||
activation_kind: int,
|
||||
) -> torch.Tensor:
|
||||
return x.new_empty((x.size(0), hidden_size))
|
||||
|
||||
|
||||
def fused_experts_cpu(
|
||||
hidden_states: torch.Tensor,
|
||||
w1: torch.Tensor,
|
||||
|
||||
@@ -14,12 +14,13 @@ from vllm.model_executor.layers.fused_moe.config import (
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
kInt4W4A8StaticChannelSym,
|
||||
kInt4W4A8StaticGroup32Sym,
|
||||
kInt4W4A8StaticGroup64Sym,
|
||||
kInt4W4A8StaticGroup128Sym,
|
||||
kInt4W4A8StaticGroupSym,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.platforms import CpuArchEnum, current_platform
|
||||
|
||||
|
||||
class CPUExpertsInt4(mk.FusedMoEExpertsMonolithic):
|
||||
@@ -48,6 +49,48 @@ class CPUExpertsInt4(mk.FusedMoEExpertsMonolithic):
|
||||
def activation_format() -> mk.FusedMoEActivationFormat:
|
||||
return mk.FusedMoEActivationFormat.Standard
|
||||
|
||||
@staticmethod
|
||||
def is_supported_config(
|
||||
cls: type[mk.FusedMoEExperts],
|
||||
moe_config: FusedMoEConfig,
|
||||
weight_key: QuantKey | None,
|
||||
activation_key: QuantKey | None,
|
||||
activation_format: mk.FusedMoEActivationFormat,
|
||||
) -> tuple[bool, str | None]:
|
||||
if (
|
||||
not current_platform.is_cpu()
|
||||
or current_platform.get_cpu_architecture() != CpuArchEnum.ARM
|
||||
):
|
||||
return False, "kernel only supports Arm CPU"
|
||||
|
||||
if moe_config.in_dtype not in (
|
||||
torch.float32,
|
||||
torch.bfloat16,
|
||||
torch.float16,
|
||||
):
|
||||
return (
|
||||
False,
|
||||
f"kernel does not support {moe_config.in_dtype} input/output dtype",
|
||||
)
|
||||
|
||||
try:
|
||||
_ = torch.ops.aten._dyn_quant_matmul_4bit
|
||||
_ = torch.ops.aten._dyn_quant_pack_4bit_weight
|
||||
except AttributeError:
|
||||
return (
|
||||
False,
|
||||
f"PyTorch {torch.__version__} does not support "
|
||||
"_dyn_quant_* 4bit ops. Install a newer version",
|
||||
)
|
||||
|
||||
return mk.FusedMoEExperts.is_supported_config(
|
||||
cls,
|
||||
moe_config,
|
||||
weight_key,
|
||||
activation_key,
|
||||
activation_format,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _supports_current_device() -> bool:
|
||||
return current_platform.is_cpu()
|
||||
@@ -86,8 +129,9 @@ class CPUExpertsInt4(mk.FusedMoEExpertsMonolithic):
|
||||
Can be channel-wise or group-wise quantization
|
||||
- Activations: dynamic per-token 8-bit integer quantization
|
||||
"""
|
||||
# group size must be multiple of 32
|
||||
# channelwise or groupwise with group size being a multiple of 32
|
||||
SUPPORTED_W_A = [
|
||||
(kInt4W4A8StaticChannelSym, None),
|
||||
(kInt4W4A8StaticGroup128Sym, None),
|
||||
(kInt4W4A8StaticGroup64Sym, None),
|
||||
(kInt4W4A8StaticGroup32Sym, None),
|
||||
@@ -120,7 +164,8 @@ class CPUExpertsInt4(mk.FusedMoEExpertsMonolithic):
|
||||
"""Expert parallelism not yet supported."""
|
||||
return False
|
||||
|
||||
def _activation_kind(self, activation: MoEActivation) -> int:
|
||||
@staticmethod
|
||||
def _activation_kind(activation: MoEActivation) -> int:
|
||||
"""Convert MoEActivation to kernel activation kind integer.
|
||||
|
||||
Returns:
|
||||
@@ -200,29 +245,22 @@ class CPUExpertsInt4(mk.FusedMoEExpertsMonolithic):
|
||||
e_score_correction_bias=e_score_correction_bias,
|
||||
)
|
||||
|
||||
# Extract dimensions from weight tensors
|
||||
# w1 is w13_packed: [num_experts, packed_data...]
|
||||
# w2 is w2_packed: [num_experts, packed_data...]
|
||||
# These dimensions should be available from the layer
|
||||
# For now, we'll extract from moe_config
|
||||
K = self.moe_config.hidden_dim
|
||||
N = self.moe_config.intermediate_size_per_partition
|
||||
hidden_size = self.moe_config.hidden_dim
|
||||
intermediate_size = self.moe_config.intermediate_size_per_partition
|
||||
assert self.quant_config.block_shape is not None
|
||||
if self.quant_config.is_per_act_token:
|
||||
# C++ kernel expects an int: -1 for channelwise, and group size for groupwise
|
||||
if self.quant_config.block_shape == [-1, 1]:
|
||||
group_size = -1
|
||||
else:
|
||||
group_size = self.quant_config.block_shape[1]
|
||||
|
||||
# Call the dynamic 4-bit int MoE kernel
|
||||
return torch.ops._C.dynamic_4bit_int_moe(
|
||||
hidden_states,
|
||||
topk_ids.to(torch.long),
|
||||
topk_weights,
|
||||
w1, # w13_weight_packed
|
||||
w2, # w2_weight_packed
|
||||
K, # hidden_size (w2_out_features)
|
||||
N, # intermediate_size (w2_in_features)
|
||||
N * 2, # 2*intermediate_size (w13_out_features)
|
||||
hidden_size,
|
||||
intermediate_size,
|
||||
group_size,
|
||||
apply_router_weight_on_input,
|
||||
self._activation_kind(activation),
|
||||
|
||||
@@ -270,7 +270,7 @@ def convert_to_w4a8_int8_moe_format(
|
||||
"""
|
||||
# Derive dimensions from tensor shapes
|
||||
E = w13_weight.shape[0] # num_experts
|
||||
I2 = w13_weight.shape[1] # w13_out_features (2*IN)
|
||||
w13_out_features = w13_weight.shape[1] # 2 * intermediate_size
|
||||
H = w13_weight.shape[2] # w13_in_features (hidden_size)
|
||||
IN = w2_weight.shape[2] # w2_in_features (intermediate_size)
|
||||
w2_out_features = w2_weight.shape[1] # Should equal H
|
||||
@@ -286,7 +286,7 @@ def convert_to_w4a8_int8_moe_format(
|
||||
w13_weight_scale[e], # [2I, H/g or 1]
|
||||
w13_bias[e] if w13_bias is not None else None, # [2I]
|
||||
H,
|
||||
I2,
|
||||
w13_out_features,
|
||||
group_size,
|
||||
)
|
||||
)
|
||||
|
||||
+7
-24
@@ -5,7 +5,6 @@
|
||||
import torch
|
||||
from compressed_tensors.quantization import (
|
||||
QuantizationArgs,
|
||||
QuantizationStrategy,
|
||||
)
|
||||
|
||||
from vllm.logger import init_logger
|
||||
@@ -32,7 +31,6 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
ScaleDesc,
|
||||
)
|
||||
from vllm.model_executor.utils import replace_parameter, set_weight_attrs
|
||||
from vllm.platforms import CpuArchEnum, current_platform
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -64,32 +62,17 @@ class CompressedTensorsW4A8Int8MoEMethod(CompressedTensorsMoEMethod):
|
||||
weight_quant.group_size if (weight_quant.group_size is not None) else -1
|
||||
)
|
||||
|
||||
# Validate scheme: weights=W4 (channel or group),
|
||||
# activations=dynamic TOKEN (A8)
|
||||
|
||||
# Must be dynamic per-token activations
|
||||
if (
|
||||
input_quant.strategy != QuantizationStrategy.TOKEN
|
||||
or not input_quant.dynamic
|
||||
# make sure group size is valid
|
||||
if self.group_size != -1 and (
|
||||
moe.hidden_dim % self.group_size != 0
|
||||
or moe.intermediate_size_per_partition % self.group_size != 0
|
||||
):
|
||||
raise ValueError(
|
||||
"W4A8-int MoE needs dynamic per-token activation quantization."
|
||||
f"Group size ({self.group_size}) must evenly divide both "
|
||||
f"hidden size ({moe.hidden_dim}) and intermediate size per "
|
||||
f"partition ({moe.intermediate_size_per_partition})."
|
||||
)
|
||||
|
||||
if weight_quant.num_bits != 4:
|
||||
raise ValueError("This method only supports 4-bit weights (num_bits=4).")
|
||||
|
||||
# Arm: check _dyn ops availability
|
||||
if current_platform.get_cpu_architecture() == CpuArchEnum.ARM:
|
||||
try:
|
||||
_ = torch.ops.aten._dyn_quant_matmul_4bit
|
||||
_ = torch.ops.aten._dyn_quant_pack_4bit_weight
|
||||
except AttributeError as err:
|
||||
raise RuntimeError(
|
||||
f"""PyTorch {torch.__version__} lacks _dyn_quant_* 4bit ops;
|
||||
install a newer build."""
|
||||
) from err
|
||||
|
||||
# Construct QuantKey for weights from QuantizationArgs
|
||||
# W4A8 INT4: 4-bit weights (stored as int8), static quantization
|
||||
if self.group_size == -1:
|
||||
|
||||
@@ -216,6 +216,12 @@ kInt4W4A8StaticGroupSym = QuantKey(
|
||||
torch.int8, kInt4W4A8StaticGroupScale, symmetric=True
|
||||
)
|
||||
|
||||
kInt4W4A8StaticChannelSym = QuantKey(
|
||||
torch.int8,
|
||||
ScaleDesc(torch.float32, True, GroupShape.PER_CHANNEL),
|
||||
symmetric=True,
|
||||
)
|
||||
|
||||
|
||||
def create_fp8_quant_key(
|
||||
static: bool,
|
||||
|
||||
Reference in New Issue
Block a user