[CPU][BugFix] Multiple fixes to w4a8_int8 CPU MoE path (#46739)

Signed-off-by: Fadi Arafeh <fadi.arafeh@arm.com>
This commit is contained in:
Fadi Arafeh
2026-07-06 05:39:20 +00:00
committed by GitHub
parent 394edc8108
commit f1073c050c
10 changed files with 189 additions and 187 deletions
@@ -5,7 +5,6 @@
import torch
from compressed_tensors.quantization import (
QuantizationArgs,
QuantizationStrategy,
)
from vllm.logger import init_logger
@@ -32,7 +31,6 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
ScaleDesc,
)
from vllm.model_executor.utils import replace_parameter, set_weight_attrs
from vllm.platforms import CpuArchEnum, current_platform
logger = init_logger(__name__)
@@ -64,32 +62,17 @@ class CompressedTensorsW4A8Int8MoEMethod(CompressedTensorsMoEMethod):
weight_quant.group_size if (weight_quant.group_size is not None) else -1
)
# Validate scheme: weights=W4 (channel or group),
# activations=dynamic TOKEN (A8)
# Must be dynamic per-token activations
if (
input_quant.strategy != QuantizationStrategy.TOKEN
or not input_quant.dynamic
# make sure group size is valid
if self.group_size != -1 and (
moe.hidden_dim % self.group_size != 0
or moe.intermediate_size_per_partition % self.group_size != 0
):
raise ValueError(
"W4A8-int MoE needs dynamic per-token activation quantization."
f"Group size ({self.group_size}) must evenly divide both "
f"hidden size ({moe.hidden_dim}) and intermediate size per "
f"partition ({moe.intermediate_size_per_partition})."
)
if weight_quant.num_bits != 4:
raise ValueError("This method only supports 4-bit weights (num_bits=4).")
# Arm: check _dyn ops availability
if current_platform.get_cpu_architecture() == CpuArchEnum.ARM:
try:
_ = torch.ops.aten._dyn_quant_matmul_4bit
_ = torch.ops.aten._dyn_quant_pack_4bit_weight
except AttributeError as err:
raise RuntimeError(
f"""PyTorch {torch.__version__} lacks _dyn_quant_* 4bit ops;
install a newer build."""
) from err
# Construct QuantKey for weights from QuantizationArgs
# W4A8 INT4: 4-bit weights (stored as int8), static quantization
if self.group_size == -1:
@@ -216,6 +216,12 @@ kInt4W4A8StaticGroupSym = QuantKey(
torch.int8, kInt4W4A8StaticGroupScale, symmetric=True
)
kInt4W4A8StaticChannelSym = QuantKey(
torch.int8,
ScaleDesc(torch.float32, True, GroupShape.PER_CHANNEL),
symmetric=True,
)
def create_fp8_quant_key(
static: bool,