[feature]Add int4 quantization support for emulation moe backend (#48451)

Signed-off-by: Qiang Li <qiang.li2@amd.com>
This commit is contained in:
qli88
2026-07-16 02:00:12 +00:00
committed by GitHub
parent 915dffaa5f
commit 5de1add806
9 changed files with 1653 additions and 10 deletions
@@ -774,6 +774,9 @@ class AutoAWQMoEMethod(FusedMoEMethodBase):
w2_bias=getattr(layer, "w2_bias", None),
a1_gscale=getattr(layer, "w13_input_global_scale", None),
a2_gscale=getattr(layer, "w2_input_global_scale", None),
gemm1_clamp_limit=getattr(layer, "swiglu_limit", None),
gemm1_alpha=getattr(layer, "swiglu_alpha", None),
gemm1_beta=getattr(layer, "swiglu_beta", None),
)
def select_gemm_impl(
@@ -142,6 +142,23 @@ class CompressedTensorsMoEMethod(FusedMoEMethodBase):
return CompressedTensorsW4A16FlydslMoEMethod(
weight_quant, input_quant, layer.moe_config
)
elif moe_backend == "emulation":
# Although this is called 'Marlin', actually it selects
# emulation backend by calling select_wna16_moe_backend.
# TODO: we need to update CompressedTensorsWNA16MoeMethod
# to honor "--moe-backend" option
from .compressed_tensors_moe_wna16_marlin import (
CompressedTensorsWNA16MarlinMoEMethod,
)
logger.info_once(
"Using CompressedTensorsWNA16MarlinMoEMethod "
"(emulation backend requested)"
)
return CompressedTensorsWNA16MarlinMoEMethod(
weight_quant, input_quant, layer.moe_config, layer_name
)
from .compressed_tensors_moe_wna16 import (
CompressedTensorsWNA16MoEMethod,
)
@@ -108,7 +108,8 @@ class CompressedTensorsW4A16FlydslMoEMethod(CompressedTensorsMoEMethod):
# grouped actorder isn't supported by this kernel
assert weight_quant.actorder != "group"
assert weight_quant.symmetric, (
"Only symmetric quantization is supported for MoE"
"Only symmetric quantization is supported for MoE. "
"Try --moe-backend emulation."
)
def create_weights(
@@ -48,7 +48,8 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
# grouped actorder isn't supported by this kernel
assert weight_quant.actorder != "group"
assert weight_quant.symmetric, (
"Only symmetric quantization is supported for MoE"
"Only symmetric quantization is supported for MoE. "
"Try --moe-backend emulation."
)
def create_weights(
@@ -467,10 +467,16 @@ class CompressedTensorsWNA16MarlinMoEMethod(CompressedTensorsMoEMethod):
# Marlin-specific parameters (not needed for Flashinfer)
if not is_flashinfer:
replace_parameter(layer, "w13_weight_g_idx", w13_g_idx_processed)
replace_parameter(layer, "w2_weight_g_idx", w2_g_idx_processed)
replace_parameter(layer, "w13_g_idx_sort_indices", w13_g_idx_sort_indices)
replace_parameter(layer, "w2_g_idx_sort_indices", w2_g_idx_sort_indices)
if w13_g_idx_processed is not None:
replace_parameter(layer, "w13_weight_g_idx", w13_g_idx_processed)
if w2_g_idx_processed is not None:
replace_parameter(layer, "w2_weight_g_idx", w2_g_idx_processed)
if w13_g_idx_sort_indices is not None:
replace_parameter(
layer, "w13_g_idx_sort_indices", w13_g_idx_sort_indices
)
if w2_g_idx_sort_indices is not None:
replace_parameter(layer, "w2_g_idx_sort_indices", w2_g_idx_sort_indices)
# Register input global scales if present
if w13_input_global_scale is not None:
@@ -484,8 +490,11 @@ class CompressedTensorsWNA16MarlinMoEMethod(CompressedTensorsMoEMethod):
torch.nn.Parameter(w2_input_global_scale, requires_grad=False),
)
if self.experts_cls is not None and issubclass(
self.experts_cls, FusedMoEExpertsModular
# Marlin workspace — only needed for Marlin-family backends, not emulation.
if (
self.experts_cls is not None
and issubclass(self.experts_cls, FusedMoEExpertsModular)
and self.wna16_backend != WNA16MoEBackend.EMULATION
):
layer.workspace = marlin_make_workspace_new(
layer.w13_weight_g_idx.device, 4
@@ -528,6 +537,9 @@ class CompressedTensorsWNA16MarlinMoEMethod(CompressedTensorsMoEMethod):
num_bits=self.num_bits,
w1_zp=getattr(layer, "w13_weight_zero_point", None),
w2_zp=getattr(layer, "w2_weight_zero_point", None),
gemm1_clamp_limit=getattr(layer, "swiglu_limit", None),
gemm1_alpha=getattr(layer, "swiglu_alpha", None),
gemm1_beta=getattr(layer, "swiglu_beta", None),
)
def apply_monolithic(