[feature]Add int4 quantization support for emulation moe backend (#48451)
Signed-off-by: Qiang Li <qiang.li2@amd.com>
This commit is contained in:
@@ -774,6 +774,9 @@ class AutoAWQMoEMethod(FusedMoEMethodBase):
|
||||
w2_bias=getattr(layer, "w2_bias", None),
|
||||
a1_gscale=getattr(layer, "w13_input_global_scale", None),
|
||||
a2_gscale=getattr(layer, "w2_input_global_scale", None),
|
||||
gemm1_clamp_limit=getattr(layer, "swiglu_limit", None),
|
||||
gemm1_alpha=getattr(layer, "swiglu_alpha", None),
|
||||
gemm1_beta=getattr(layer, "swiglu_beta", None),
|
||||
)
|
||||
|
||||
def select_gemm_impl(
|
||||
|
||||
+17
@@ -142,6 +142,23 @@ class CompressedTensorsMoEMethod(FusedMoEMethodBase):
|
||||
return CompressedTensorsW4A16FlydslMoEMethod(
|
||||
weight_quant, input_quant, layer.moe_config
|
||||
)
|
||||
elif moe_backend == "emulation":
|
||||
# Although this is called 'Marlin', actually it selects
|
||||
# emulation backend by calling select_wna16_moe_backend.
|
||||
# TODO: we need to update CompressedTensorsWNA16MoeMethod
|
||||
# to honor "--moe-backend" option
|
||||
from .compressed_tensors_moe_wna16_marlin import (
|
||||
CompressedTensorsWNA16MarlinMoEMethod,
|
||||
)
|
||||
|
||||
logger.info_once(
|
||||
"Using CompressedTensorsWNA16MarlinMoEMethod "
|
||||
"(emulation backend requested)"
|
||||
)
|
||||
return CompressedTensorsWNA16MarlinMoEMethod(
|
||||
weight_quant, input_quant, layer.moe_config, layer_name
|
||||
)
|
||||
|
||||
from .compressed_tensors_moe_wna16 import (
|
||||
CompressedTensorsWNA16MoEMethod,
|
||||
)
|
||||
|
||||
+2
-1
@@ -108,7 +108,8 @@ class CompressedTensorsW4A16FlydslMoEMethod(CompressedTensorsMoEMethod):
|
||||
# grouped actorder isn't supported by this kernel
|
||||
assert weight_quant.actorder != "group"
|
||||
assert weight_quant.symmetric, (
|
||||
"Only symmetric quantization is supported for MoE"
|
||||
"Only symmetric quantization is supported for MoE. "
|
||||
"Try --moe-backend emulation."
|
||||
)
|
||||
|
||||
def create_weights(
|
||||
|
||||
+2
-1
@@ -48,7 +48,8 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
|
||||
# grouped actorder isn't supported by this kernel
|
||||
assert weight_quant.actorder != "group"
|
||||
assert weight_quant.symmetric, (
|
||||
"Only symmetric quantization is supported for MoE"
|
||||
"Only symmetric quantization is supported for MoE. "
|
||||
"Try --moe-backend emulation."
|
||||
)
|
||||
|
||||
def create_weights(
|
||||
|
||||
+18
-6
@@ -467,10 +467,16 @@ class CompressedTensorsWNA16MarlinMoEMethod(CompressedTensorsMoEMethod):
|
||||
|
||||
# Marlin-specific parameters (not needed for Flashinfer)
|
||||
if not is_flashinfer:
|
||||
replace_parameter(layer, "w13_weight_g_idx", w13_g_idx_processed)
|
||||
replace_parameter(layer, "w2_weight_g_idx", w2_g_idx_processed)
|
||||
replace_parameter(layer, "w13_g_idx_sort_indices", w13_g_idx_sort_indices)
|
||||
replace_parameter(layer, "w2_g_idx_sort_indices", w2_g_idx_sort_indices)
|
||||
if w13_g_idx_processed is not None:
|
||||
replace_parameter(layer, "w13_weight_g_idx", w13_g_idx_processed)
|
||||
if w2_g_idx_processed is not None:
|
||||
replace_parameter(layer, "w2_weight_g_idx", w2_g_idx_processed)
|
||||
if w13_g_idx_sort_indices is not None:
|
||||
replace_parameter(
|
||||
layer, "w13_g_idx_sort_indices", w13_g_idx_sort_indices
|
||||
)
|
||||
if w2_g_idx_sort_indices is not None:
|
||||
replace_parameter(layer, "w2_g_idx_sort_indices", w2_g_idx_sort_indices)
|
||||
|
||||
# Register input global scales if present
|
||||
if w13_input_global_scale is not None:
|
||||
@@ -484,8 +490,11 @@ class CompressedTensorsWNA16MarlinMoEMethod(CompressedTensorsMoEMethod):
|
||||
torch.nn.Parameter(w2_input_global_scale, requires_grad=False),
|
||||
)
|
||||
|
||||
if self.experts_cls is not None and issubclass(
|
||||
self.experts_cls, FusedMoEExpertsModular
|
||||
# Marlin workspace — only needed for Marlin-family backends, not emulation.
|
||||
if (
|
||||
self.experts_cls is not None
|
||||
and issubclass(self.experts_cls, FusedMoEExpertsModular)
|
||||
and self.wna16_backend != WNA16MoEBackend.EMULATION
|
||||
):
|
||||
layer.workspace = marlin_make_workspace_new(
|
||||
layer.w13_weight_g_idx.device, 4
|
||||
@@ -528,6 +537,9 @@ class CompressedTensorsWNA16MarlinMoEMethod(CompressedTensorsMoEMethod):
|
||||
num_bits=self.num_bits,
|
||||
w1_zp=getattr(layer, "w13_weight_zero_point", None),
|
||||
w2_zp=getattr(layer, "w2_weight_zero_point", None),
|
||||
gemm1_clamp_limit=getattr(layer, "swiglu_limit", None),
|
||||
gemm1_alpha=getattr(layer, "swiglu_alpha", None),
|
||||
gemm1_beta=getattr(layer, "swiglu_beta", None),
|
||||
)
|
||||
|
||||
def apply_monolithic(
|
||||
|
||||
Reference in New Issue
Block a user