[CPU][BugFix] Multiple fixes to w4a8_int8 CPU MoE path (#46739)
Signed-off-by: Fadi Arafeh <fadi.arafeh@arm.com>
This commit is contained in:
+7
-24
@@ -5,7 +5,6 @@
|
||||
import torch
|
||||
from compressed_tensors.quantization import (
|
||||
QuantizationArgs,
|
||||
QuantizationStrategy,
|
||||
)
|
||||
|
||||
from vllm.logger import init_logger
|
||||
@@ -32,7 +31,6 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
ScaleDesc,
|
||||
)
|
||||
from vllm.model_executor.utils import replace_parameter, set_weight_attrs
|
||||
from vllm.platforms import CpuArchEnum, current_platform
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -64,32 +62,17 @@ class CompressedTensorsW4A8Int8MoEMethod(CompressedTensorsMoEMethod):
|
||||
weight_quant.group_size if (weight_quant.group_size is not None) else -1
|
||||
)
|
||||
|
||||
# Validate scheme: weights=W4 (channel or group),
|
||||
# activations=dynamic TOKEN (A8)
|
||||
|
||||
# Must be dynamic per-token activations
|
||||
if (
|
||||
input_quant.strategy != QuantizationStrategy.TOKEN
|
||||
or not input_quant.dynamic
|
||||
# make sure group size is valid
|
||||
if self.group_size != -1 and (
|
||||
moe.hidden_dim % self.group_size != 0
|
||||
or moe.intermediate_size_per_partition % self.group_size != 0
|
||||
):
|
||||
raise ValueError(
|
||||
"W4A8-int MoE needs dynamic per-token activation quantization."
|
||||
f"Group size ({self.group_size}) must evenly divide both "
|
||||
f"hidden size ({moe.hidden_dim}) and intermediate size per "
|
||||
f"partition ({moe.intermediate_size_per_partition})."
|
||||
)
|
||||
|
||||
if weight_quant.num_bits != 4:
|
||||
raise ValueError("This method only supports 4-bit weights (num_bits=4).")
|
||||
|
||||
# Arm: check _dyn ops availability
|
||||
if current_platform.get_cpu_architecture() == CpuArchEnum.ARM:
|
||||
try:
|
||||
_ = torch.ops.aten._dyn_quant_matmul_4bit
|
||||
_ = torch.ops.aten._dyn_quant_pack_4bit_weight
|
||||
except AttributeError as err:
|
||||
raise RuntimeError(
|
||||
f"""PyTorch {torch.__version__} lacks _dyn_quant_* 4bit ops;
|
||||
install a newer build."""
|
||||
) from err
|
||||
|
||||
# Construct QuantKey for weights from QuantizationArgs
|
||||
# W4A8 INT4: 4-bit weights (stored as int8), static quantization
|
||||
if self.group_size == -1:
|
||||
|
||||
@@ -216,6 +216,12 @@ kInt4W4A8StaticGroupSym = QuantKey(
|
||||
torch.int8, kInt4W4A8StaticGroupScale, symmetric=True
|
||||
)
|
||||
|
||||
kInt4W4A8StaticChannelSym = QuantKey(
|
||||
torch.int8,
|
||||
ScaleDesc(torch.float32, True, GroupShape.PER_CHANNEL),
|
||||
symmetric=True,
|
||||
)
|
||||
|
||||
|
||||
def create_fp8_quant_key(
|
||||
static: bool,
|
||||
|
||||
Reference in New Issue
Block a user