[CPU][BugFix] Multiple fixes to w4a8_int8 CPU MoE path (#46739)
Signed-off-by: Fadi Arafeh <fadi.arafeh@arm.com>
This commit is contained in:
@@ -14,12 +14,13 @@ from vllm.model_executor.layers.fused_moe.config import (
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
kInt4W4A8StaticChannelSym,
|
||||
kInt4W4A8StaticGroup32Sym,
|
||||
kInt4W4A8StaticGroup64Sym,
|
||||
kInt4W4A8StaticGroup128Sym,
|
||||
kInt4W4A8StaticGroupSym,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.platforms import CpuArchEnum, current_platform
|
||||
|
||||
|
||||
class CPUExpertsInt4(mk.FusedMoEExpertsMonolithic):
|
||||
@@ -48,6 +49,48 @@ class CPUExpertsInt4(mk.FusedMoEExpertsMonolithic):
|
||||
def activation_format() -> mk.FusedMoEActivationFormat:
|
||||
return mk.FusedMoEActivationFormat.Standard
|
||||
|
||||
@staticmethod
|
||||
def is_supported_config(
|
||||
cls: type[mk.FusedMoEExperts],
|
||||
moe_config: FusedMoEConfig,
|
||||
weight_key: QuantKey | None,
|
||||
activation_key: QuantKey | None,
|
||||
activation_format: mk.FusedMoEActivationFormat,
|
||||
) -> tuple[bool, str | None]:
|
||||
if (
|
||||
not current_platform.is_cpu()
|
||||
or current_platform.get_cpu_architecture() != CpuArchEnum.ARM
|
||||
):
|
||||
return False, "kernel only supports Arm CPU"
|
||||
|
||||
if moe_config.in_dtype not in (
|
||||
torch.float32,
|
||||
torch.bfloat16,
|
||||
torch.float16,
|
||||
):
|
||||
return (
|
||||
False,
|
||||
f"kernel does not support {moe_config.in_dtype} input/output dtype",
|
||||
)
|
||||
|
||||
try:
|
||||
_ = torch.ops.aten._dyn_quant_matmul_4bit
|
||||
_ = torch.ops.aten._dyn_quant_pack_4bit_weight
|
||||
except AttributeError:
|
||||
return (
|
||||
False,
|
||||
f"PyTorch {torch.__version__} does not support "
|
||||
"_dyn_quant_* 4bit ops. Install a newer version",
|
||||
)
|
||||
|
||||
return mk.FusedMoEExperts.is_supported_config(
|
||||
cls,
|
||||
moe_config,
|
||||
weight_key,
|
||||
activation_key,
|
||||
activation_format,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _supports_current_device() -> bool:
|
||||
return current_platform.is_cpu()
|
||||
@@ -86,8 +129,9 @@ class CPUExpertsInt4(mk.FusedMoEExpertsMonolithic):
|
||||
Can be channel-wise or group-wise quantization
|
||||
- Activations: dynamic per-token 8-bit integer quantization
|
||||
"""
|
||||
# group size must be multiple of 32
|
||||
# channelwise or groupwise with group size being a multiple of 32
|
||||
SUPPORTED_W_A = [
|
||||
(kInt4W4A8StaticChannelSym, None),
|
||||
(kInt4W4A8StaticGroup128Sym, None),
|
||||
(kInt4W4A8StaticGroup64Sym, None),
|
||||
(kInt4W4A8StaticGroup32Sym, None),
|
||||
@@ -120,7 +164,8 @@ class CPUExpertsInt4(mk.FusedMoEExpertsMonolithic):
|
||||
"""Expert parallelism not yet supported."""
|
||||
return False
|
||||
|
||||
def _activation_kind(self, activation: MoEActivation) -> int:
|
||||
@staticmethod
|
||||
def _activation_kind(activation: MoEActivation) -> int:
|
||||
"""Convert MoEActivation to kernel activation kind integer.
|
||||
|
||||
Returns:
|
||||
@@ -200,29 +245,22 @@ class CPUExpertsInt4(mk.FusedMoEExpertsMonolithic):
|
||||
e_score_correction_bias=e_score_correction_bias,
|
||||
)
|
||||
|
||||
# Extract dimensions from weight tensors
|
||||
# w1 is w13_packed: [num_experts, packed_data...]
|
||||
# w2 is w2_packed: [num_experts, packed_data...]
|
||||
# These dimensions should be available from the layer
|
||||
# For now, we'll extract from moe_config
|
||||
K = self.moe_config.hidden_dim
|
||||
N = self.moe_config.intermediate_size_per_partition
|
||||
hidden_size = self.moe_config.hidden_dim
|
||||
intermediate_size = self.moe_config.intermediate_size_per_partition
|
||||
assert self.quant_config.block_shape is not None
|
||||
if self.quant_config.is_per_act_token:
|
||||
# C++ kernel expects an int: -1 for channelwise, and group size for groupwise
|
||||
if self.quant_config.block_shape == [-1, 1]:
|
||||
group_size = -1
|
||||
else:
|
||||
group_size = self.quant_config.block_shape[1]
|
||||
|
||||
# Call the dynamic 4-bit int MoE kernel
|
||||
return torch.ops._C.dynamic_4bit_int_moe(
|
||||
hidden_states,
|
||||
topk_ids.to(torch.long),
|
||||
topk_weights,
|
||||
w1, # w13_weight_packed
|
||||
w2, # w2_weight_packed
|
||||
K, # hidden_size (w2_out_features)
|
||||
N, # intermediate_size (w2_in_features)
|
||||
N * 2, # 2*intermediate_size (w13_out_features)
|
||||
hidden_size,
|
||||
intermediate_size,
|
||||
group_size,
|
||||
apply_router_weight_on_input,
|
||||
self._activation_kind(activation),
|
||||
|
||||
@@ -270,7 +270,7 @@ def convert_to_w4a8_int8_moe_format(
|
||||
"""
|
||||
# Derive dimensions from tensor shapes
|
||||
E = w13_weight.shape[0] # num_experts
|
||||
I2 = w13_weight.shape[1] # w13_out_features (2*IN)
|
||||
w13_out_features = w13_weight.shape[1] # 2 * intermediate_size
|
||||
H = w13_weight.shape[2] # w13_in_features (hidden_size)
|
||||
IN = w2_weight.shape[2] # w2_in_features (intermediate_size)
|
||||
w2_out_features = w2_weight.shape[1] # Should equal H
|
||||
@@ -286,7 +286,7 @@ def convert_to_w4a8_int8_moe_format(
|
||||
w13_weight_scale[e], # [2I, H/g or 1]
|
||||
w13_bias[e] if w13_bias is not None else None, # [2I]
|
||||
H,
|
||||
I2,
|
||||
w13_out_features,
|
||||
group_size,
|
||||
)
|
||||
)
|
||||
|
||||
+7
-24
@@ -5,7 +5,6 @@
|
||||
import torch
|
||||
from compressed_tensors.quantization import (
|
||||
QuantizationArgs,
|
||||
QuantizationStrategy,
|
||||
)
|
||||
|
||||
from vllm.logger import init_logger
|
||||
@@ -32,7 +31,6 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
ScaleDesc,
|
||||
)
|
||||
from vllm.model_executor.utils import replace_parameter, set_weight_attrs
|
||||
from vllm.platforms import CpuArchEnum, current_platform
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -64,32 +62,17 @@ class CompressedTensorsW4A8Int8MoEMethod(CompressedTensorsMoEMethod):
|
||||
weight_quant.group_size if (weight_quant.group_size is not None) else -1
|
||||
)
|
||||
|
||||
# Validate scheme: weights=W4 (channel or group),
|
||||
# activations=dynamic TOKEN (A8)
|
||||
|
||||
# Must be dynamic per-token activations
|
||||
if (
|
||||
input_quant.strategy != QuantizationStrategy.TOKEN
|
||||
or not input_quant.dynamic
|
||||
# make sure group size is valid
|
||||
if self.group_size != -1 and (
|
||||
moe.hidden_dim % self.group_size != 0
|
||||
or moe.intermediate_size_per_partition % self.group_size != 0
|
||||
):
|
||||
raise ValueError(
|
||||
"W4A8-int MoE needs dynamic per-token activation quantization."
|
||||
f"Group size ({self.group_size}) must evenly divide both "
|
||||
f"hidden size ({moe.hidden_dim}) and intermediate size per "
|
||||
f"partition ({moe.intermediate_size_per_partition})."
|
||||
)
|
||||
|
||||
if weight_quant.num_bits != 4:
|
||||
raise ValueError("This method only supports 4-bit weights (num_bits=4).")
|
||||
|
||||
# Arm: check _dyn ops availability
|
||||
if current_platform.get_cpu_architecture() == CpuArchEnum.ARM:
|
||||
try:
|
||||
_ = torch.ops.aten._dyn_quant_matmul_4bit
|
||||
_ = torch.ops.aten._dyn_quant_pack_4bit_weight
|
||||
except AttributeError as err:
|
||||
raise RuntimeError(
|
||||
f"""PyTorch {torch.__version__} lacks _dyn_quant_* 4bit ops;
|
||||
install a newer build."""
|
||||
) from err
|
||||
|
||||
# Construct QuantKey for weights from QuantizationArgs
|
||||
# W4A8 INT4: 4-bit weights (stored as int8), static quantization
|
||||
if self.group_size == -1:
|
||||
|
||||
@@ -216,6 +216,12 @@ kInt4W4A8StaticGroupSym = QuantKey(
|
||||
torch.int8, kInt4W4A8StaticGroupScale, symmetric=True
|
||||
)
|
||||
|
||||
kInt4W4A8StaticChannelSym = QuantKey(
|
||||
torch.int8,
|
||||
ScaleDesc(torch.float32, True, GroupShape.PER_CHANNEL),
|
||||
symmetric=True,
|
||||
)
|
||||
|
||||
|
||||
def create_fp8_quant_key(
|
||||
static: bool,
|
||||
|
||||
Reference in New Issue
Block a user