Improve Mistral format checks. (#33253)

Signed-off-by: Julien Denize <julien.denize@mistral.ai>
Signed-off-by: juliendenize <julien.denize@mistral.ai>
Co-authored-by: Michael Goin <mgoin64@gmail.com>
This commit is contained in:
Julien Denize
2026-01-30 06:23:33 -08:00
committed by GitHub
co-authored by Michael Goin
parent a11bc12d53
commit ae5b7aff2b
8 changed files with 193 additions and 24 deletions
+6 -2
View File
@@ -23,6 +23,7 @@ from transformers.utils import CONFIG_NAME as HF_CONFIG_NAME
from vllm import envs
from vllm.logger import init_logger
from vllm.transformers_utils.repo_utils import is_mistral_model_repo
from vllm.transformers_utils.utils import parse_safetensors_file_metadata
from .config_parser_base import ConfigParserBase
@@ -49,7 +50,6 @@ except ImportError:
ALLOWED_LAYER_TYPES as ALLOWED_ATTENTION_LAYER_TYPES,
)
if envs.VLLM_USE_MODELSCOPE:
from modelscope import AutoConfig
else:
@@ -581,7 +581,11 @@ def get_config(
try:
# First check for Mistral to avoid defaulting to
# Transformers implementation.
if file_or_path_exists(model, MISTRAL_CONFIG_NAME, revision=revision):
if is_mistral_model_repo(
model_name_or_path=str(model), revision=revision
) and file_or_path_exists(
model=model, config_name=MISTRAL_CONFIG_NAME, revision=revision
):
config_format = "mistral"
elif (_is_gguf and not _is_remote_gguf) or file_or_path_exists(
model, HF_CONFIG_NAME, revision=revision
@@ -1,9 +1,12 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Iterator
from contextlib import contextmanager
from typing import final
import torch
from huggingface_hub import constants
from safetensors.torch import _TYPES as _SAFETENSORS_TO_TORCH_DTYPE
from transformers import PretrainedConfig
@@ -14,6 +17,7 @@ from vllm.config.model_arch import (
from vllm.config.utils import getattr_iter
from vllm.logger import init_logger
from vllm.transformers_utils.config import (
ConfigFormat,
try_get_safetensors_metadata,
)
from vllm.utils.torch_utils import common_broadcastable_dtype
@@ -21,6 +25,22 @@ from vllm.utils.torch_utils import common_broadcastable_dtype
logger = init_logger(__name__)
@contextmanager
def _maybe_patch_hf_hub_constants(config_format: ConfigFormat) -> Iterator[None]:
if config_format == "mistral":
hf_safetensors_single_file = constants.SAFETENSORS_SINGLE_FILE
hf_safetensors_index_file = constants.SAFETENSORS_INDEX_FILE
constants.SAFETENSORS_SINGLE_FILE = "consolidated.safetensors"
constants.SAFETENSORS_INDEX_FILE = "consolidated.safetensors.index.json"
try:
yield
finally:
constants.SAFETENSORS_SINGLE_FILE = hf_safetensors_single_file
constants.SAFETENSORS_INDEX_FILE = hf_safetensors_index_file
else:
yield
class ModelArchConfigConvertorBase:
def __init__(self, hf_config: PretrainedConfig, hf_text_config: PretrainedConfig):
self.hf_config = hf_config
@@ -123,7 +143,11 @@ class ModelArchConfigConvertorBase:
@final
@classmethod
def get_torch_dtype(
cls, hf_config: PretrainedConfig, model_id: str, revision: str | None
cls,
hf_config: PretrainedConfig,
model_id: str,
revision: str | None,
config_format: ConfigFormat,
):
# NOTE: getattr(config, "dtype", torch.float32) is not correct
# because config.dtype can be None.
@@ -140,7 +164,8 @@ class ModelArchConfigConvertorBase:
# Try to read the dtype of the weights if they are in safetensors format
if config_dtype is None:
repo_mt = try_get_safetensors_metadata(model_id, revision=revision)
with _maybe_patch_hf_hub_constants(config_format):
repo_mt = try_get_safetensors_metadata(model_id, revision=revision)
if repo_mt and (files_mt := repo_mt.files_metadata):
param_dtypes: set[torch.dtype] = {
+36
View File
@@ -127,6 +127,42 @@ def list_filtered_repo_files(
return file_list
def any_pattern_in_repo_files(
model_name_or_path: str,
allow_patterns: list[str],
revision: str | None = None,
repo_type: str | None = None,
token: str | bool | None = None,
):
return (
len(
list_filtered_repo_files(
model_name_or_path=model_name_or_path,
allow_patterns=allow_patterns,
revision=revision,
repo_type=repo_type,
token=token,
)
)
> 0
)
def is_mistral_model_repo(
model_name_or_path: str,
revision: str | None = None,
repo_type: str | None = None,
token: str | bool | None = None,
) -> bool:
return any_pattern_in_repo_files(
model_name_or_path=model_name_or_path,
allow_patterns=["consolidated*.safetensors"],
revision=revision,
repo_type=repo_type,
token=token,
)
def file_exists(
repo_id: str,
file_name: str,