[feature] MiniMax-M3-MXFP4 support added (#45896)

Signed-off-by: Qiang Li <qiang.li2@amd.com>
This commit is contained in:
qli88
2026-06-17 18:50:48 +00:00
committed by GitHub
parent 2a47a9ff0f
commit d112eb1ac7
4 changed files with 44 additions and 3 deletions
@@ -755,6 +755,7 @@ class UnfusedOAITritonExperts(LoRAExpertsMixin, BaseOAITritonExperts):
MoEActivation.GELU,
MoEActivation.SWIGLUOAI,
MoEActivation.SWIGLUSTEP,
MoEActivation.SWIGLUOAI_UNINTERLEAVE,
]
@staticmethod
@@ -811,6 +812,19 @@ class UnfusedOAITritonExperts(LoRAExpertsMixin, BaseOAITritonExperts):
input,
quant_config.gemm1_clamp_limit,
)
elif activation == MoEActivation.SWIGLUOAI_UNINTERLEAVE:
assert quant_config.gemm1_clamp_limit is not None
alpha = (
quant_config.gemm1_alpha
if quant_config.gemm1_alpha is not None
else 1.0
)
beta = (
quant_config.gemm1_beta if quant_config.gemm1_beta is not None else 0.0
)
torch.ops._C.silu_and_mul_with_clamp(
output, input, quant_config.gemm1_clamp_limit, alpha, beta
)
else:
super().activation(activation, output, input)
@@ -503,7 +503,18 @@ def select_mxfp4_moe_backend(
activation_format,
)
if current_platform.is_cuda() or current_platform.is_rocm():
if current_platform.is_rocm():
backend = Mxfp4MoeBackend.TRITON_UNFUSED
logger.info_once(_make_log_backend(backend))
return _return_or_raise(
Mxfp4MoeBackend.TRITON_UNFUSED,
config,
kMxfp4Static,
None,
activation_format,
)
if current_platform.is_cuda():
raise NotImplementedError(
"No MXFP4 MoE backend supports the deployment configuration. "
f"weight_key=kMxfp4Static, activation_key={activation_key}. "
@@ -1303,6 +1303,9 @@ class QuarkOCP_MX_MoEMethod(QuarkMoEMethod):
w2_bias=getattr(layer, "w2_bias", None),
a1_scale=getattr(layer, "w13_input_scale", None),
a2_scale=getattr(layer, "w2_input_scale", None),
gemm1_alpha=getattr(layer, "swiglu_alpha", None),
gemm1_beta=getattr(layer, "swiglu_beta", None),
swiglu_limit=getattr(layer, "swiglu_limit", None),
)
# Emulation and other schemes
@@ -1339,6 +1342,9 @@ class QuarkOCP_MX_MoEMethod(QuarkMoEMethod):
a1_scale=None,
a2_scale=None,
block_shape=None,
gemm1_alpha=getattr(layer, "swiglu_alpha", None),
gemm1_beta=getattr(layer, "swiglu_beta", None),
gemm1_clamp_limit=getattr(layer, "swiglu_limit", None),
)
@property