From 02bf9c7907b491146e2a79c3839af243c4478e00 Mon Sep 17 00:00:00 2001 From: xiao-llm Date: Thu, 16 Jul 2026 14:58:47 -0400 Subject: [PATCH] Fix Quark mxfp4 quantized model loading issue under mtp (#46757) Signed-off-by: Xiao Yu Co-authored-by: Andreas Karatzas --- .../layers/quantization/quark/quark.py | 12 ++++++++++-- .../layers/quantization/quark/utils.py | 16 ++++++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/vllm/model_executor/layers/quantization/quark/quark.py b/vllm/model_executor/layers/quantization/quark/quark.py index fbd61e28cd..694116827c 100644 --- a/vllm/model_executor/layers/quantization/quark/quark.py +++ b/vllm/model_executor/layers/quantization/quark/quark.py @@ -9,7 +9,10 @@ from transformers import PretrainedConfig from vllm.logger import init_logger from vllm.model_executor.layers.attention import Attention -from vllm.model_executor.layers.fused_moe import RoutedExperts +from vllm.model_executor.layers.fused_moe import ( + RoutedExperts, + UnquantizedFusedMoEMethod, +) from vllm.model_executor.layers.linear import ( LinearBase, LinearMethodBase, @@ -146,8 +149,13 @@ class QuarkConfig(QuantizationConfig): # Check if the layer is skipped for quantization. exclude_layers = cast(list[str], self.quant_config.get("exclude")) if should_ignore_layer( - prefix, ignore=exclude_layers, fused_mapping=self.packed_modules_mapping + prefix, + ignore=exclude_layers, + fused_mapping=self.packed_modules_mapping, + check_children=isinstance(layer, RoutedExperts), ): + if isinstance(layer, RoutedExperts): + return UnquantizedFusedMoEMethod(layer.moe_config) if ( "self_attn" not in prefix # only quantize attention projections or not getattr(self, "dynamic_mxfp4_quant", False) diff --git a/vllm/model_executor/layers/quantization/quark/utils.py b/vllm/model_executor/layers/quantization/quark/utils.py index ee55e5d39e..8f6ce4d91c 100644 --- a/vllm/model_executor/layers/quantization/quark/utils.py +++ b/vllm/model_executor/layers/quantization/quark/utils.py @@ -27,10 +27,26 @@ def should_ignore_layer( layer_name: str | None, ignore: Iterable[str], fused_mapping: Mapping[str, list[str]] = MappingProxyType({}), + *, + check_children: bool = False, ) -> bool: if layer_name is None: return False + # MoE layers are currently all-or-nothing: if any child is ignored, + # the parent layer must be ignored as well. For example, the + # amd/GLM-5.2-MXFP4 config ignores children like + # model.layers.78.mlp.experts.*.down_proj, while the layer checked + # here is the parent model.layers.N.mlp.experts. + # See: + # https://huggingface.co/amd/GLM-5.2-MXFP4/blob/main/config.json#L793-L795 + if check_children and any( + target == layer_name or target.startswith(layer_name + ".") + for target in ignore + if not target.startswith("re:") + ): + return True + # layer_name = model.layers.0.self_attn.qkv_proj # proj_name = qkv_proj proj_name = layer_name.split(".")[-1]