[ROCm][Bugfix] fix exception related to trust_remote_code for MiniMax-M2.1-MXFP4 (#37698)

Signed-off-by: Hongxia Yang <hongxiay.yang@amd.com>
Co-authored-by: Hongxia Yang <hongxiay.yang@amd.com>
This commit is contained in:
Hongxia Yang
2026-03-30 15:49:23 +00:00
committed by GitHub
co-authored by Hongxia Yang
parent e8b055a5ac
commit dbdd9ae067
9 changed files with 142 additions and 19 deletions
@@ -5,6 +5,7 @@ import fnmatch
from typing import TYPE_CHECKING, Any, cast
import torch
from transformers import PretrainedConfig
from vllm.logger import init_logger
from vllm.model_executor.layers.attention import Attention
@@ -36,7 +37,6 @@ from vllm.model_executor.layers.quantization.quark.utils import (
)
from vllm.model_executor.models.utils import WeightsMapper
from vllm.platforms import current_platform
from vllm.transformers_utils.config import get_config
if TYPE_CHECKING:
from vllm.model_executor.models.utils import WeightsMapper
@@ -45,6 +45,10 @@ __all__ = ["QuarkLinearMethod"]
logger = init_logger(__name__)
# model_type values that use dynamic MXFP4 re-quantization for
# OCP MX fp4 Quark checkpoints
_DEEPSEEK_V3_FAMILY_MODEL_TYPES = frozenset({"deepseek_v3"})
class QuarkConfig(QuantizationConfig):
def __init__(
@@ -63,19 +67,28 @@ class QuarkConfig(QuantizationConfig):
self.pack_method = pack_method
self.dynamic_mxfp4_quant = False
def maybe_update_config(self, model_name: str, revision: str | None = None):
self.hf_config = get_config(
model=model_name,
trust_remote_code=False, # or get from model_config if available
revision=revision,
config_format="auto",
)
def maybe_update_config(
self,
model_name: str,
hf_config: PretrainedConfig | None = None,
revision: str | None = None,
):
"""Enable dynamic MXFP4 only for DeepSeek-V3-family + fp4 Quark checkpoints."""
quant_config = getattr(self.hf_config, "quantization_config", None)
if (
getattr(hf_config, "model_type", None)
not in _DEEPSEEK_V3_FAMILY_MODEL_TYPES
):
return
quant_config = getattr(hf_config, "quantization_config", None)
if quant_config is not None:
quant_dtype = quant_config["global_quant_config"]["weight"]["dtype"]
model_type = self.hf_config.model_type
if quant_dtype == "fp4" and model_type == "deepseek_v3":
quant_dtype = (
quant_config.get("global_quant_config", {})
.get("weight", {})
.get("dtype")
)
if quant_dtype == "fp4":
self.dynamic_mxfp4_quant = True
def get_linear_method(self) -> "QuarkLinearMethod":