Fix Quark mxfp4 quantized model loading issue under mtp (#46757)
Signed-off-by: Xiao Yu <xiao.yu.dc@outlook.com> Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
co-authored by
Andreas Karatzas
parent
f61163e6c7
commit
02bf9c7907
@@ -9,7 +9,10 @@ from transformers import PretrainedConfig
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import RoutedExperts
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
RoutedExperts,
|
||||
UnquantizedFusedMoEMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.linear import (
|
||||
LinearBase,
|
||||
LinearMethodBase,
|
||||
@@ -146,8 +149,13 @@ class QuarkConfig(QuantizationConfig):
|
||||
# Check if the layer is skipped for quantization.
|
||||
exclude_layers = cast(list[str], self.quant_config.get("exclude"))
|
||||
if should_ignore_layer(
|
||||
prefix, ignore=exclude_layers, fused_mapping=self.packed_modules_mapping
|
||||
prefix,
|
||||
ignore=exclude_layers,
|
||||
fused_mapping=self.packed_modules_mapping,
|
||||
check_children=isinstance(layer, RoutedExperts),
|
||||
):
|
||||
if isinstance(layer, RoutedExperts):
|
||||
return UnquantizedFusedMoEMethod(layer.moe_config)
|
||||
if (
|
||||
"self_attn" not in prefix # only quantize attention projections
|
||||
or not getattr(self, "dynamic_mxfp4_quant", False)
|
||||
|
||||
Reference in New Issue
Block a user