[compressed-tensors] Asymmetric support for MoE WNA16 marlin (#44025)
Signed-off-by: Brian Dellabetta <bdellabe@redhat.com>
This commit is contained in:
@@ -429,6 +429,30 @@ def maybe_warn_marlin_atomic_add(device, dtype):
|
||||
)
|
||||
|
||||
|
||||
def moe_packed_to_marlin_zero_points(
|
||||
q_zp_packed: torch.Tensor,
|
||||
size_k: int,
|
||||
size_n: int,
|
||||
num_bits: int,
|
||||
is_a_8bit: bool = False,
|
||||
):
|
||||
"""Convert compressed-tensors packed zero points to Marlin format.
|
||||
|
||||
Unlike AWQ, compressed-tensors uses standard bit packing without
|
||||
interleaving, so we just unpack and apply Marlin permutation directly.
|
||||
"""
|
||||
num_experts = q_zp_packed.shape[0]
|
||||
output = torch.empty(
|
||||
(num_experts, q_zp_packed.shape[1], q_zp_packed.shape[2]),
|
||||
device=q_zp_packed.device,
|
||||
dtype=q_zp_packed.dtype,
|
||||
)
|
||||
for e in range(num_experts):
|
||||
q_zp = unpack_cols(q_zp_packed[e], num_bits, size_k, size_n)
|
||||
output[e] = marlin_zero_points(q_zp, size_k, size_n, num_bits, is_a_8bit)
|
||||
return output
|
||||
|
||||
|
||||
def maybe_warn_marlin_atomic_add_env():
|
||||
if torch.compiler.is_dynamo_compiling():
|
||||
return
|
||||
|
||||
@@ -181,6 +181,13 @@ kInt8Static = QuantKey(INT8_DTYPE, scale=kInt8StaticGroupScale, symmetric=True)
|
||||
kInt4Static32GroupScale = ScaleDesc(torch.float16, True, GroupShape(1, 32))
|
||||
kInt4Static32 = QuantKey(INT4_DTYPE, scale=kInt4Static32GroupScale, symmetric=True)
|
||||
|
||||
kInt4StaticAsym = QuantKey(
|
||||
scalar_types.uint4, scale=kInt4StaticGroupScale, symmetric=False
|
||||
)
|
||||
kInt4Static32Asym = QuantKey(
|
||||
scalar_types.uint4, scale=kInt4Static32GroupScale, symmetric=False
|
||||
)
|
||||
|
||||
kInt8StaticChannelSym = QuantKey(torch.int8, kStaticChannelScale, symmetric=True)
|
||||
kInt8DynamicTokenSym = QuantKey(torch.int8, kDynamicTokenScale, symmetric=True)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user