[Quant] Use canonical N-first weight format for CT WNA16 MoE (#52798)

Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
Co-authored-by: HDCharles <{"message":"Not Found","documentation_url":"https://docs.github.com/rest/users/emails#list-email-addresses-for-the-authenticated-user","status":"404"}>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
HDCharles
2026-09-28 11:22:44 -06:00
committed by GitHub
co-authored by Claude Opus 4.6 HDCharles <{"message":"Not Found","documentation_url":"https://docs.github.com/rest/users/emails#list-email-addresses-for-the-authenticated-user","status":"404"}> OpenAI Codex OpenAI Codex
parent 6d3ea3c2c9
commit c01368fd64
6 changed files with 222 additions and 209 deletions
@@ -626,19 +626,31 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
layer.register_parameter("w2_bias", None)
w2_bias = None
# Normalize GPTQ K-first layout to canonical N-first format.
w13 = layer.w13_qweight.data.transpose(1, 2).contiguous()
w2 = layer.w2_qweight.data.transpose(1, 2).contiguous()
w13_scale = layer.w13_scales.data.transpose(1, 2).contiguous()
w2_scale = layer.w2_scales.data.transpose(1, 2).contiguous()
w13_qzeros = getattr(layer, "w13_qzeros", None)
w2_qzeros = getattr(layer, "w2_qzeros", None)
if w13_qzeros is not None:
w13_qzeros = w13_qzeros.data.transpose(1, 2).contiguous()
if w2_qzeros is not None:
w2_qzeros = w2_qzeros.data.transpose(1, 2).contiguous()
converted = convert_to_wna16_moe_kernel_format(
backend=self.wna16_moe_backend,
layer=layer,
quant_config=self.quant_config,
input_dtype=self.input_dtype,
w13=layer.w13_qweight,
w2=layer.w2_qweight,
w13_scale=layer.w13_scales,
w2_scale=layer.w2_scales,
w13=w13,
w2=w2,
w13_scale=w13_scale,
w2_scale=w2_scale,
w13_bias=w13_bias,
w2_bias=w2_bias,
w13_qzeros=getattr(layer, "w13_qzeros", None),
w2_qzeros=getattr(layer, "w2_qzeros", None),
w13_qzeros=w13_qzeros,
w2_qzeros=w2_qzeros,
)
if converted is None:
@@ -112,10 +112,10 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
WNA16MoEBackend.MARLIN,
WNA16MoEBackend.BATCHED_MARLIN,
]
self.is_transposed = self.wna16_backend not in (
WNA16MoEBackend.FLASHINFER_TRTLLM,
WNA16MoEBackend.HUMMING,
)
# CT checkpoints always load in N-first format [E, N, K_packed].
# Backend-specific transposition is handled in
# convert_to_wna16_moe_kernel_format.
self.is_transposed = False
if self.is_marlin:
assert check_moe_marlin_supports_config(
@@ -141,80 +141,31 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
num_groups_w2: int | None = None,
num_groups_w13: int | None = None,
) -> tuple[int, int, int]:
"""Get the shape of the weight based on the weight name, number of experts
hidden size, intermediate size per partition, number of groups for w2,
and number of groups for w13. Pass in num_groups_w2 and num_groups_w13
for weight scales/zero_points.
"""Return N-first buffer shape for the given weight/scale/zp tensor.
All CT MoE weights are allocated in the checkpoint's native N-first
layout ``[E, N, K_packed]``. Backend-specific transposition (e.g. to
Marlin's K-first layout) is deferred to
``convert_to_wna16_moe_kernel_format``.
"""
if weight_name in ("w13_scale", "w13_zp"):
assert num_groups_w13 is not None, (
"num_groups_w13 must be provided for weight scales/zero_points"
)
if weight_name in ("w2_scale", "w2_zp"):
assert num_groups_w2 is not None, (
"num_groups_w2 must be provided for weight scales/zero_points"
)
w13_num_shards = 2 if self.moe.is_act_and_mul else 1
shape_map: dict[str, dict[str, tuple[int, int | None, int | None]]] = {
"w13_weight": {
"Flashinfer": (
num_experts,
w13_num_shards * intermediate_size_per_partition,
self._packed_dim(hidden_size),
),
"Marlin": (
num_experts,
self._packed_dim(hidden_size),
w13_num_shards * intermediate_size_per_partition,
),
},
"w13_scale": {
"Flashinfer": (
num_experts,
w13_num_shards * intermediate_size_per_partition,
num_groups_w13,
),
"Marlin": (
num_experts,
num_groups_w13,
w13_num_shards * intermediate_size_per_partition,
),
},
"w13_zp": {
"Marlin": (
num_experts,
num_groups_w13,
self._packed_dim(w13_num_shards * intermediate_size_per_partition),
),
},
"w2_weight": {
"Flashinfer": (
num_experts,
hidden_size,
self._packed_dim(intermediate_size_per_partition),
),
"Marlin": (
num_experts,
self._packed_dim(intermediate_size_per_partition),
hidden_size,
),
},
"w2_scale": {
"Flashinfer": (num_experts, hidden_size, num_groups_w2),
"Marlin": (num_experts, num_groups_w2, hidden_size),
},
"w2_zp": {
"Marlin": (
num_experts,
num_groups_w2,
self._packed_dim(hidden_size),
),
},
w13_n = w13_num_shards * intermediate_size_per_partition
pf = self.packed_factor
shape_map = {
"w13_weight": (num_experts, w13_n, hidden_size // pf),
"w13_scale": (num_experts, w13_n, num_groups_w13),
"w13_zp": (num_experts, w13_n // pf, num_groups_w13),
"w2_weight": (
num_experts,
hidden_size,
intermediate_size_per_partition // pf,
),
"w2_scale": (num_experts, hidden_size, num_groups_w2),
"w2_zp": (num_experts, hidden_size // pf, num_groups_w2),
}
backend_key = "Marlin" if self.is_transposed else "Flashinfer"
shape = shape_map[weight_name][backend_key]
assert shape[1] is not None and shape[2] is not None
return shape[0], shape[1], shape[2]
shape = shape_map[weight_name]
assert isinstance(shape, tuple) and all(isinstance(d, int) for d in shape)
return shape # type: ignore[return-value]
def create_weights(
self,
@@ -225,11 +176,8 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
params_dtype: torch.dtype,
**extra_weight_attrs,
):
# Will transpose the loaded weight along the
# intermediate and hidden dim sizes. Will
# shard for TP along the transposed dims
extra_weight_attrs.update(
{"is_transposed": self.is_transposed, "quant_method": self.strategy}
{"is_transposed": False, "quant_method": self.strategy}
)
w13_weight = torch.nn.Parameter(
@@ -440,12 +388,10 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
replace_parameter(layer, "w13_weight_scale", w13_scales)
replace_parameter(layer, "w2_weight_scale", w2_scales)
# CPU fused_experts_cpu and the RDNA3 HIP kernel require zero points
# even for symmetric quant (the oracle synthesizes them).
# CPU fused_experts_cpu requires zero points even for symmetric quant.
# EMULATION bakes ZP into the dequantized bf16 weights — ZP is None.
if (
not self.symmetric
or self.wna16_backend in (WNA16MoEBackend.CPU, WNA16MoEBackend.RDNA3)
not self.symmetric or self.wna16_backend == WNA16MoEBackend.CPU
) and self.wna16_backend != WNA16MoEBackend.EMULATION:
assert w13_qzeros is not None and w2_qzeros is not None
replace_parameter(layer, "w13_weight_zero_point", w13_qzeros)