Fix Quark mxfp4 quantized model loading issue under mtp (#46757)

Signed-off-by: Xiao Yu <xiao.yu.dc@outlook.com>
Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
xiao-llm
2026-07-16 14:58:47 -04:00
committed by GitHub
co-authored by Andreas Karatzas
parent f61163e6c7
commit 02bf9c7907
2 changed files with 26 additions and 2 deletions
@@ -9,7 +9,10 @@ from transformers import PretrainedConfig
from vllm.logger import init_logger
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import RoutedExperts
from vllm.model_executor.layers.fused_moe import (
RoutedExperts,
UnquantizedFusedMoEMethod,
)
from vllm.model_executor.layers.linear import (
LinearBase,
LinearMethodBase,
@@ -146,8 +149,13 @@ class QuarkConfig(QuantizationConfig):
# Check if the layer is skipped for quantization.
exclude_layers = cast(list[str], self.quant_config.get("exclude"))
if should_ignore_layer(
prefix, ignore=exclude_layers, fused_mapping=self.packed_modules_mapping
prefix,
ignore=exclude_layers,
fused_mapping=self.packed_modules_mapping,
check_children=isinstance(layer, RoutedExperts),
):
if isinstance(layer, RoutedExperts):
return UnquantizedFusedMoEMethod(layer.moe_config)
if (
"self_attn" not in prefix # only quantize attention projections
or not getattr(self, "dynamic_mxfp4_quant", False)