[compressed-tensors] Asymmetric support for MoE WNA16 marlin (#44025)

Signed-off-by: Brian Dellabetta <bdellabe@redhat.com>
This commit is contained in:
Brian Dellabetta
2026-06-02 08:51:45 -07:00
committed by GitHub
parent 53fa09d085
commit 774e552397
5 changed files with 126 additions and 12 deletions
@@ -429,6 +429,30 @@ def maybe_warn_marlin_atomic_add(device, dtype):
)
def moe_packed_to_marlin_zero_points(
q_zp_packed: torch.Tensor,
size_k: int,
size_n: int,
num_bits: int,
is_a_8bit: bool = False,
):
"""Convert compressed-tensors packed zero points to Marlin format.
Unlike AWQ, compressed-tensors uses standard bit packing without
interleaving, so we just unpack and apply Marlin permutation directly.
"""
num_experts = q_zp_packed.shape[0]
output = torch.empty(
(num_experts, q_zp_packed.shape[1], q_zp_packed.shape[2]),
device=q_zp_packed.device,
dtype=q_zp_packed.dtype,
)
for e in range(num_experts):
q_zp = unpack_cols(q_zp_packed[e], num_bits, size_k, size_n)
output[e] = marlin_zero_points(q_zp, size_k, size_n, num_bits, is_a_8bit)
return output
def maybe_warn_marlin_atomic_add_env():
if torch.compiler.is_dynamo_compiling():
return
@@ -181,6 +181,13 @@ kInt8Static = QuantKey(INT8_DTYPE, scale=kInt8StaticGroupScale, symmetric=True)
kInt4Static32GroupScale = ScaleDesc(torch.float16, True, GroupShape(1, 32))
kInt4Static32 = QuantKey(INT4_DTYPE, scale=kInt4Static32GroupScale, symmetric=True)
kInt4StaticAsym = QuantKey(
scalar_types.uint4, scale=kInt4StaticGroupScale, symmetric=False
)
kInt4Static32Asym = QuantKey(
scalar_types.uint4, scale=kInt4Static32GroupScale, symmetric=False
)
kInt8StaticChannelSym = QuantKey(torch.int8, kStaticChannelScale, symmetric=True)
kInt8DynamicTokenSym = QuantKey(torch.int8, kDynamicTokenScale, symmetric=True)