[Bugfix][ROCm] Fix a few int4/int8 quantization errors (#52112)
Signed-off-by: Qiang Li <qiang.li2@amd.com>
This commit is contained in:
@@ -48,6 +48,8 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kFp8StaticTensorSym,
|
||||
kInt4Static,
|
||||
kInt4Static32,
|
||||
kInt4Static32Asym,
|
||||
kInt4StaticAsym,
|
||||
kInt8DynamicTensorSym,
|
||||
kInt8DynamicTokenSym,
|
||||
kInt8Static,
|
||||
@@ -186,8 +188,8 @@ class TritonExperts(LoRAExpertsMixin, mk.FusedMoEExpertsModular):
|
||||
swiglu_limit_func(output, input, activation_config.clamp_limit)
|
||||
return
|
||||
|
||||
# SWIGLUOAI_UNINTERLEAVE routes to the silu_and_mul_with_clamp kernel and
|
||||
# requires a clamp limit. Other activations ignore these parameters.
|
||||
# SWIGLUOAI_UNINTERLEAVE routes to torch.ops._C.silu_and_mul_with_clamp
|
||||
# via apply_moe_activation() and requires clamp_limit to be set.
|
||||
if activation == MoEActivation.SWIGLUOAI_UNINTERLEAVE:
|
||||
assert activation_config.clamp_limit is not None, (
|
||||
"SWIGLUOAI_UNINTERLEAVE requires gemm1_clamp_limit"
|
||||
@@ -580,6 +582,8 @@ class TritonWNA16Experts(TritonExperts):
|
||||
kInt4Static,
|
||||
kInt8Static,
|
||||
kInt4Static32,
|
||||
kInt4StaticAsym,
|
||||
kInt4Static32Asym,
|
||||
# other group sizes?
|
||||
]
|
||||
return weight_key in SUPPORTED_W
|
||||
@@ -591,6 +595,7 @@ class TritonWNA16Experts(TritonExperts):
|
||||
MoEActivation.GELU,
|
||||
MoEActivation.GELU_TANH,
|
||||
MoEActivation.SWIGLUOAI,
|
||||
MoEActivation.SWIGLUOAI_UNINTERLEAVE,
|
||||
MoEActivation.SWIGLUSTEP,
|
||||
MoEActivation.SILU_NO_MUL,
|
||||
MoEActivation.GELU_NO_MUL,
|
||||
|
||||
@@ -1649,6 +1649,42 @@ def convert_to_wna16_moe_kernel_format(
|
||||
w2_uint8 = w2.transpose(1, 2).contiguous().view(torch.uint8)
|
||||
w13_scale = w13_scale.transpose(1, 2).contiguous()
|
||||
w2_scale = w2_scale.transpose(1, 2).contiguous()
|
||||
# Zero points from compressed-tensors checkpoints are K-first int32
|
||||
# with 8 int4 ZPs packed per element: shape (E, K//gs, N//8).
|
||||
# fused_moe_kernel_gptq_awq expects N-first uint8 with 2 int4 ZPs
|
||||
# per byte: shape (E, N//2, K//gs), indexed as
|
||||
# (offs_bn // 2) * stride_bzn + offs_k_group * stride_bzk.
|
||||
# Conversion steps:
|
||||
# (E, K//gs, N//8) int32
|
||||
# → transpose(1,2) → (E, N//8, K//gs) int32
|
||||
# → view(uint8) → (E, N//8, K//gs*4) [each int32 → 4 bytes]
|
||||
# → reshape(…,4) → (E, N//8, K//gs, 4) [isolate byte index]
|
||||
# → permute(0,1,3,2) → (E, N//8, 4, K//gs) [byte index before K]
|
||||
# → reshape → (E, N//2, K//gs) [kernel expected layout]
|
||||
# After this, element [e, offs_bn//2, k_group] is the uint8 byte
|
||||
# holding the two int4 ZPs for output channels offs_bn and offs_bn+1.
|
||||
if w13_qzeros is not None:
|
||||
E13, Kg13, Np13 = w13_qzeros.shape
|
||||
w13_qzeros = (
|
||||
w13_qzeros.transpose(1, 2)
|
||||
.contiguous()
|
||||
.view(torch.uint8)
|
||||
.reshape(E13, Np13, Kg13, 4)
|
||||
.permute(0, 1, 3, 2)
|
||||
.reshape(E13, Np13 * 4, Kg13)
|
||||
.contiguous()
|
||||
)
|
||||
if w2_qzeros is not None:
|
||||
E2, Kg2, Np2 = w2_qzeros.shape
|
||||
w2_qzeros = (
|
||||
w2_qzeros.transpose(1, 2)
|
||||
.contiguous()
|
||||
.view(torch.uint8)
|
||||
.reshape(E2, Np2, Kg2, 4)
|
||||
.permute(0, 1, 3, 2)
|
||||
.reshape(E2, Np2 * 4, Kg2)
|
||||
.contiguous()
|
||||
)
|
||||
else:
|
||||
# MoeWNA16 uses N-first uint8 weights and scales.
|
||||
w13_uint8 = w13.view(torch.uint8)
|
||||
|
||||
+26
-4
@@ -123,8 +123,6 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
|
||||
# grouped actorder isn't supported by this kernel
|
||||
assert weight_quant.actorder != "group"
|
||||
|
||||
assert self.symmetric, "Only symmetric quantization is supported for MoE"
|
||||
|
||||
# Non-Marlin WNA16 always uses bf16/fp16 inputs
|
||||
self.input_dtype = torch.bfloat16
|
||||
|
||||
@@ -317,6 +315,27 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
|
||||
num_groups_w2 = w2_scales_size // self.group_size
|
||||
num_groups_w13 = hidden_size // self.group_size
|
||||
|
||||
if not self.symmetric:
|
||||
# For asymmetric int4 quantization, packed_factor (=8) int4 ZPs are
|
||||
# stored per int32. Each TP shard must contain a whole number of
|
||||
# int32 elements so the packed ZP can be sliced without straddling
|
||||
# an int32 boundary.
|
||||
w13_n = 2 * intermediate_size_per_partition # gate+up output channels
|
||||
if w13_n % self.packed_factor != 0:
|
||||
raise ValueError(
|
||||
f"CompressedTensors WNA16 MoE: gate+up output channels per "
|
||||
f"TP rank (2 * intermediate_size_per_partition = {w13_n}) "
|
||||
f"must be divisible by packed_factor ({self.packed_factor}). "
|
||||
f"Use a TP size where 2 * intermediate_size is divisible by "
|
||||
f"{self.packed_factor}."
|
||||
)
|
||||
if hidden_size % self.packed_factor != 0:
|
||||
raise ValueError(
|
||||
f"CompressedTensors WNA16 MoE: hidden_size ({hidden_size}) "
|
||||
f"must be divisible by packed_factor ({self.packed_factor}) "
|
||||
f"for correct ZP unpacking."
|
||||
)
|
||||
|
||||
layer.num_groups_w13 = num_groups_w13
|
||||
layer.num_groups_w2 = num_groups_w2
|
||||
|
||||
@@ -513,8 +532,11 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
|
||||
replace_parameter(layer, "w13_weight_scale", w13_scales)
|
||||
replace_parameter(layer, "w2_weight_scale", w2_scales)
|
||||
|
||||
# CPU fused_experts_cpu requires zero points even for symmetric quant
|
||||
if not self.symmetric or self.wna16_backend == WNA16MoEBackend.CPU:
|
||||
# CPU fused_experts_cpu requires zero points even for symmetric quant.
|
||||
# EMULATION bakes ZP into the dequantized bf16 weights — ZP is None.
|
||||
if (
|
||||
not self.symmetric or self.wna16_backend == WNA16MoEBackend.CPU
|
||||
) and self.wna16_backend != WNA16MoEBackend.EMULATION:
|
||||
assert w13_qzeros is not None and w2_qzeros is not None
|
||||
replace_parameter(layer, "w13_weight_zero_point", w13_qzeros)
|
||||
replace_parameter(layer, "w2_weight_zero_point", w2_qzeros)
|
||||
|
||||
@@ -222,7 +222,6 @@ class MoeWNA16Method(FusedMoEMethodBase):
|
||||
else:
|
||||
scale = kInt4StaticGroupScale
|
||||
elif num_bits == 8:
|
||||
assert group_size == -1
|
||||
quant_type = INT8_DTYPE
|
||||
scale = kInt8StaticGroupScale
|
||||
else:
|
||||
|
||||
Reference in New Issue
Block a user