[Bugfix][ROCm] Fix a few int4/int8 quantization errors (#52112)

Signed-off-by: Qiang Li <qiang.li2@amd.com>
This commit is contained in:
qli88
2026-08-18 08:31:51 +00:00
committed by GitHub
parent 5c9ff5366b
commit e8ad2855e7
4 changed files with 69 additions and 7 deletions
@@ -48,6 +48,8 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
kFp8StaticTensorSym,
kInt4Static,
kInt4Static32,
kInt4Static32Asym,
kInt4StaticAsym,
kInt8DynamicTensorSym,
kInt8DynamicTokenSym,
kInt8Static,
@@ -186,8 +188,8 @@ class TritonExperts(LoRAExpertsMixin, mk.FusedMoEExpertsModular):
swiglu_limit_func(output, input, activation_config.clamp_limit)
return
# SWIGLUOAI_UNINTERLEAVE routes to the silu_and_mul_with_clamp kernel and
# requires a clamp limit. Other activations ignore these parameters.
# SWIGLUOAI_UNINTERLEAVE routes to torch.ops._C.silu_and_mul_with_clamp
# via apply_moe_activation() and requires clamp_limit to be set.
if activation == MoEActivation.SWIGLUOAI_UNINTERLEAVE:
assert activation_config.clamp_limit is not None, (
"SWIGLUOAI_UNINTERLEAVE requires gemm1_clamp_limit"
@@ -580,6 +582,8 @@ class TritonWNA16Experts(TritonExperts):
kInt4Static,
kInt8Static,
kInt4Static32,
kInt4StaticAsym,
kInt4Static32Asym,
# other group sizes?
]
return weight_key in SUPPORTED_W
@@ -591,6 +595,7 @@ class TritonWNA16Experts(TritonExperts):
MoEActivation.GELU,
MoEActivation.GELU_TANH,
MoEActivation.SWIGLUOAI,
MoEActivation.SWIGLUOAI_UNINTERLEAVE,
MoEActivation.SWIGLUSTEP,
MoEActivation.SILU_NO_MUL,
MoEActivation.GELU_NO_MUL,
@@ -1649,6 +1649,42 @@ def convert_to_wna16_moe_kernel_format(
w2_uint8 = w2.transpose(1, 2).contiguous().view(torch.uint8)
w13_scale = w13_scale.transpose(1, 2).contiguous()
w2_scale = w2_scale.transpose(1, 2).contiguous()
# Zero points from compressed-tensors checkpoints are K-first int32
# with 8 int4 ZPs packed per element: shape (E, K//gs, N//8).
# fused_moe_kernel_gptq_awq expects N-first uint8 with 2 int4 ZPs
# per byte: shape (E, N//2, K//gs), indexed as
# (offs_bn // 2) * stride_bzn + offs_k_group * stride_bzk.
# Conversion steps:
# (E, K//gs, N//8) int32
# → transpose(1,2) → (E, N//8, K//gs) int32
# → view(uint8) → (E, N//8, K//gs*4) [each int32 → 4 bytes]
# → reshape(…,4) → (E, N//8, K//gs, 4) [isolate byte index]
# → permute(0,1,3,2) → (E, N//8, 4, K//gs) [byte index before K]
# → reshape → (E, N//2, K//gs) [kernel expected layout]
# After this, element [e, offs_bn//2, k_group] is the uint8 byte
# holding the two int4 ZPs for output channels offs_bn and offs_bn+1.
if w13_qzeros is not None:
E13, Kg13, Np13 = w13_qzeros.shape
w13_qzeros = (
w13_qzeros.transpose(1, 2)
.contiguous()
.view(torch.uint8)
.reshape(E13, Np13, Kg13, 4)
.permute(0, 1, 3, 2)
.reshape(E13, Np13 * 4, Kg13)
.contiguous()
)
if w2_qzeros is not None:
E2, Kg2, Np2 = w2_qzeros.shape
w2_qzeros = (
w2_qzeros.transpose(1, 2)
.contiguous()
.view(torch.uint8)
.reshape(E2, Np2, Kg2, 4)
.permute(0, 1, 3, 2)
.reshape(E2, Np2 * 4, Kg2)
.contiguous()
)
else:
# MoeWNA16 uses N-first uint8 weights and scales.
w13_uint8 = w13.view(torch.uint8)
@@ -123,8 +123,6 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
# grouped actorder isn't supported by this kernel
assert weight_quant.actorder != "group"
assert self.symmetric, "Only symmetric quantization is supported for MoE"
# Non-Marlin WNA16 always uses bf16/fp16 inputs
self.input_dtype = torch.bfloat16
@@ -317,6 +315,27 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
num_groups_w2 = w2_scales_size // self.group_size
num_groups_w13 = hidden_size // self.group_size
if not self.symmetric:
# For asymmetric int4 quantization, packed_factor (=8) int4 ZPs are
# stored per int32. Each TP shard must contain a whole number of
# int32 elements so the packed ZP can be sliced without straddling
# an int32 boundary.
w13_n = 2 * intermediate_size_per_partition # gate+up output channels
if w13_n % self.packed_factor != 0:
raise ValueError(
f"CompressedTensors WNA16 MoE: gate+up output channels per "
f"TP rank (2 * intermediate_size_per_partition = {w13_n}) "
f"must be divisible by packed_factor ({self.packed_factor}). "
f"Use a TP size where 2 * intermediate_size is divisible by "
f"{self.packed_factor}."
)
if hidden_size % self.packed_factor != 0:
raise ValueError(
f"CompressedTensors WNA16 MoE: hidden_size ({hidden_size}) "
f"must be divisible by packed_factor ({self.packed_factor}) "
f"for correct ZP unpacking."
)
layer.num_groups_w13 = num_groups_w13
layer.num_groups_w2 = num_groups_w2
@@ -513,8 +532,11 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
replace_parameter(layer, "w13_weight_scale", w13_scales)
replace_parameter(layer, "w2_weight_scale", w2_scales)
# CPU fused_experts_cpu requires zero points even for symmetric quant
if not self.symmetric or self.wna16_backend == WNA16MoEBackend.CPU:
# CPU fused_experts_cpu requires zero points even for symmetric quant.
# EMULATION bakes ZP into the dequantized bf16 weights — ZP is None.
if (
not self.symmetric or self.wna16_backend == WNA16MoEBackend.CPU
) and self.wna16_backend != WNA16MoEBackend.EMULATION:
assert w13_qzeros is not None and w2_qzeros is not None
replace_parameter(layer, "w13_weight_zero_point", w13_qzeros)
replace_parameter(layer, "w2_weight_zero_point", w2_qzeros)
@@ -222,7 +222,6 @@ class MoeWNA16Method(FusedMoEMethodBase):
else:
scale = kInt4StaticGroupScale
elif num_bits == 8:
assert group_size == -1
quant_type = INT8_DTYPE
scale = kInt8StaticGroupScale
else: