[Quant] Use canonical N-first weight format for CT WNA16 MoE (#52798)
Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com> Co-authored-by: HDCharles <{"message":"Not Found","documentation_url":"https://docs.github.com/rest/users/emails#list-email-addresses-for-the-authenticated-user","status":"404"}> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
Claude Opus 4.6
HDCharles <{"message":"Not Found","documentation_url":"https://docs.github.com/rest/users/emails#list-email-addresses-for-the-authenticated-user","status":"404"}>
OpenAI Codex
OpenAI Codex
parent
6d3ea3c2c9
commit
c01368fd64
@@ -626,19 +626,31 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
|
||||
layer.register_parameter("w2_bias", None)
|
||||
w2_bias = None
|
||||
|
||||
# Normalize GPTQ K-first layout to canonical N-first format.
|
||||
w13 = layer.w13_qweight.data.transpose(1, 2).contiguous()
|
||||
w2 = layer.w2_qweight.data.transpose(1, 2).contiguous()
|
||||
w13_scale = layer.w13_scales.data.transpose(1, 2).contiguous()
|
||||
w2_scale = layer.w2_scales.data.transpose(1, 2).contiguous()
|
||||
w13_qzeros = getattr(layer, "w13_qzeros", None)
|
||||
w2_qzeros = getattr(layer, "w2_qzeros", None)
|
||||
if w13_qzeros is not None:
|
||||
w13_qzeros = w13_qzeros.data.transpose(1, 2).contiguous()
|
||||
if w2_qzeros is not None:
|
||||
w2_qzeros = w2_qzeros.data.transpose(1, 2).contiguous()
|
||||
|
||||
converted = convert_to_wna16_moe_kernel_format(
|
||||
backend=self.wna16_moe_backend,
|
||||
layer=layer,
|
||||
quant_config=self.quant_config,
|
||||
input_dtype=self.input_dtype,
|
||||
w13=layer.w13_qweight,
|
||||
w2=layer.w2_qweight,
|
||||
w13_scale=layer.w13_scales,
|
||||
w2_scale=layer.w2_scales,
|
||||
w13=w13,
|
||||
w2=w2,
|
||||
w13_scale=w13_scale,
|
||||
w2_scale=w2_scale,
|
||||
w13_bias=w13_bias,
|
||||
w2_bias=w2_bias,
|
||||
w13_qzeros=getattr(layer, "w13_qzeros", None),
|
||||
w2_qzeros=getattr(layer, "w2_qzeros", None),
|
||||
w13_qzeros=w13_qzeros,
|
||||
w2_qzeros=w2_qzeros,
|
||||
)
|
||||
|
||||
if converted is None:
|
||||
|
||||
+29
-83
@@ -112,10 +112,10 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
|
||||
WNA16MoEBackend.MARLIN,
|
||||
WNA16MoEBackend.BATCHED_MARLIN,
|
||||
]
|
||||
self.is_transposed = self.wna16_backend not in (
|
||||
WNA16MoEBackend.FLASHINFER_TRTLLM,
|
||||
WNA16MoEBackend.HUMMING,
|
||||
)
|
||||
# CT checkpoints always load in N-first format [E, N, K_packed].
|
||||
# Backend-specific transposition is handled in
|
||||
# convert_to_wna16_moe_kernel_format.
|
||||
self.is_transposed = False
|
||||
|
||||
if self.is_marlin:
|
||||
assert check_moe_marlin_supports_config(
|
||||
@@ -141,80 +141,31 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
|
||||
num_groups_w2: int | None = None,
|
||||
num_groups_w13: int | None = None,
|
||||
) -> tuple[int, int, int]:
|
||||
"""Get the shape of the weight based on the weight name, number of experts
|
||||
hidden size, intermediate size per partition, number of groups for w2,
|
||||
and number of groups for w13. Pass in num_groups_w2 and num_groups_w13
|
||||
for weight scales/zero_points.
|
||||
"""Return N-first buffer shape for the given weight/scale/zp tensor.
|
||||
|
||||
All CT MoE weights are allocated in the checkpoint's native N-first
|
||||
layout ``[E, N, K_packed]``. Backend-specific transposition (e.g. to
|
||||
Marlin's K-first layout) is deferred to
|
||||
``convert_to_wna16_moe_kernel_format``.
|
||||
"""
|
||||
if weight_name in ("w13_scale", "w13_zp"):
|
||||
assert num_groups_w13 is not None, (
|
||||
"num_groups_w13 must be provided for weight scales/zero_points"
|
||||
)
|
||||
if weight_name in ("w2_scale", "w2_zp"):
|
||||
assert num_groups_w2 is not None, (
|
||||
"num_groups_w2 must be provided for weight scales/zero_points"
|
||||
)
|
||||
w13_num_shards = 2 if self.moe.is_act_and_mul else 1
|
||||
shape_map: dict[str, dict[str, tuple[int, int | None, int | None]]] = {
|
||||
"w13_weight": {
|
||||
"Flashinfer": (
|
||||
num_experts,
|
||||
w13_num_shards * intermediate_size_per_partition,
|
||||
self._packed_dim(hidden_size),
|
||||
),
|
||||
"Marlin": (
|
||||
num_experts,
|
||||
self._packed_dim(hidden_size),
|
||||
w13_num_shards * intermediate_size_per_partition,
|
||||
),
|
||||
},
|
||||
"w13_scale": {
|
||||
"Flashinfer": (
|
||||
num_experts,
|
||||
w13_num_shards * intermediate_size_per_partition,
|
||||
num_groups_w13,
|
||||
),
|
||||
"Marlin": (
|
||||
num_experts,
|
||||
num_groups_w13,
|
||||
w13_num_shards * intermediate_size_per_partition,
|
||||
),
|
||||
},
|
||||
"w13_zp": {
|
||||
"Marlin": (
|
||||
num_experts,
|
||||
num_groups_w13,
|
||||
self._packed_dim(w13_num_shards * intermediate_size_per_partition),
|
||||
),
|
||||
},
|
||||
"w2_weight": {
|
||||
"Flashinfer": (
|
||||
num_experts,
|
||||
hidden_size,
|
||||
self._packed_dim(intermediate_size_per_partition),
|
||||
),
|
||||
"Marlin": (
|
||||
num_experts,
|
||||
self._packed_dim(intermediate_size_per_partition),
|
||||
hidden_size,
|
||||
),
|
||||
},
|
||||
"w2_scale": {
|
||||
"Flashinfer": (num_experts, hidden_size, num_groups_w2),
|
||||
"Marlin": (num_experts, num_groups_w2, hidden_size),
|
||||
},
|
||||
"w2_zp": {
|
||||
"Marlin": (
|
||||
num_experts,
|
||||
num_groups_w2,
|
||||
self._packed_dim(hidden_size),
|
||||
),
|
||||
},
|
||||
w13_n = w13_num_shards * intermediate_size_per_partition
|
||||
pf = self.packed_factor
|
||||
shape_map = {
|
||||
"w13_weight": (num_experts, w13_n, hidden_size // pf),
|
||||
"w13_scale": (num_experts, w13_n, num_groups_w13),
|
||||
"w13_zp": (num_experts, w13_n // pf, num_groups_w13),
|
||||
"w2_weight": (
|
||||
num_experts,
|
||||
hidden_size,
|
||||
intermediate_size_per_partition // pf,
|
||||
),
|
||||
"w2_scale": (num_experts, hidden_size, num_groups_w2),
|
||||
"w2_zp": (num_experts, hidden_size // pf, num_groups_w2),
|
||||
}
|
||||
backend_key = "Marlin" if self.is_transposed else "Flashinfer"
|
||||
shape = shape_map[weight_name][backend_key]
|
||||
assert shape[1] is not None and shape[2] is not None
|
||||
return shape[0], shape[1], shape[2]
|
||||
shape = shape_map[weight_name]
|
||||
assert isinstance(shape, tuple) and all(isinstance(d, int) for d in shape)
|
||||
return shape # type: ignore[return-value]
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
@@ -225,11 +176,8 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
|
||||
params_dtype: torch.dtype,
|
||||
**extra_weight_attrs,
|
||||
):
|
||||
# Will transpose the loaded weight along the
|
||||
# intermediate and hidden dim sizes. Will
|
||||
# shard for TP along the transposed dims
|
||||
extra_weight_attrs.update(
|
||||
{"is_transposed": self.is_transposed, "quant_method": self.strategy}
|
||||
{"is_transposed": False, "quant_method": self.strategy}
|
||||
)
|
||||
|
||||
w13_weight = torch.nn.Parameter(
|
||||
@@ -440,12 +388,10 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
|
||||
replace_parameter(layer, "w13_weight_scale", w13_scales)
|
||||
replace_parameter(layer, "w2_weight_scale", w2_scales)
|
||||
|
||||
# CPU fused_experts_cpu and the RDNA3 HIP kernel require zero points
|
||||
# even for symmetric quant (the oracle synthesizes them).
|
||||
# CPU fused_experts_cpu requires zero points even for symmetric quant.
|
||||
# EMULATION bakes ZP into the dequantized bf16 weights — ZP is None.
|
||||
if (
|
||||
not self.symmetric
|
||||
or self.wna16_backend in (WNA16MoEBackend.CPU, WNA16MoEBackend.RDNA3)
|
||||
not self.symmetric or self.wna16_backend == WNA16MoEBackend.CPU
|
||||
) and self.wna16_backend != WNA16MoEBackend.EMULATION:
|
||||
assert w13_qzeros is not None and w2_qzeros is not None
|
||||
replace_parameter(layer, "w13_weight_zero_point", w13_qzeros)
|
||||
|
||||
Reference in New Issue
Block a user