Improve Mistral format checks. (#33253)

Signed-off-by: Julien Denize <julien.denize@mistral.ai>
Signed-off-by: juliendenize <julien.denize@mistral.ai>
Co-authored-by: Michael Goin <mgoin64@gmail.com>
This commit is contained in:
Julien Denize
2026-01-30 06:23:33 -08:00
committed by GitHub
co-authored by Michael Goin
parent a11bc12d53
commit ae5b7aff2b
8 changed files with 193 additions and 24 deletions
+3 -1
View File
@@ -565,6 +565,7 @@ class ModelConfig:
self.dtype,
is_pooling_model=self.runner_type == "pooling",
revision=self.revision,
config_format=self.config_format,
)
self.original_max_model_len = self.max_model_len
@@ -1844,9 +1845,10 @@ def _get_and_verify_dtype(
*,
is_pooling_model: bool,
revision: str | None = None,
config_format: ConfigFormat = "hf",
) -> torch.dtype:
config_dtype = ModelArchConfigConvertorBase.get_torch_dtype(
config, model_id, revision=revision
config, model_id, revision=revision, config_format=config_format
)
model_type = config.model_type
+19 -17
View File
@@ -1,6 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import importlib.util
from dataclasses import dataclass, field
from functools import lru_cache
from pathlib import Path
@@ -18,7 +17,10 @@ from vllm.transformers_utils.gguf_utils import (
is_remote_gguf,
split_remote_gguf,
)
from vllm.transformers_utils.repo_utils import list_filtered_repo_files
from vllm.transformers_utils.repo_utils import (
any_pattern_in_repo_files,
is_mistral_model_repo,
)
from vllm.utils.import_utils import resolve_obj_by_qualname
from .protocol import TokenizerLike
@@ -142,26 +144,26 @@ def resolve_tokenizer_args(
kwargs["use_fast"] = False
# Try to use official Mistral tokenizer if possible
if tokenizer_mode == "auto" and importlib.util.find_spec("mistral_common"):
allow_patterns = ["tekken.json", "tokenizer.model.v*"]
files_list = list_filtered_repo_files(
if (
tokenizer_mode == "auto"
and is_mistral_model_repo(
model_name_or_path=str(tokenizer_name), revision=revision
)
and any_pattern_in_repo_files(
model_name_or_path=str(tokenizer_name),
allow_patterns=allow_patterns,
allow_patterns=["tekken.json", "tokenizer.model.v*"],
revision=revision,
)
if len(files_list) > 0:
tokenizer_mode = "mistral"
):
tokenizer_mode = "mistral"
# Try to use Grok2 tiktoken tokenizer if possible
if tokenizer_mode == "auto":
allow_patterns = ["tokenizer.tok.json"]
files_list = list_filtered_repo_files(
model_name_or_path=str(tokenizer_name),
allow_patterns=allow_patterns,
revision=revision,
)
if len(files_list) > 0:
tokenizer_mode = "grok2"
if tokenizer_mode == "auto" and any_pattern_in_repo_files(
model_name_or_path=str(tokenizer_name),
allow_patterns=["tokenizer.tok.json"],
revision=revision,
):
tokenizer_mode = "grok2"
# Fallback to HF tokenizer
if tokenizer_mode == "auto":
+6 -2
View File
@@ -23,6 +23,7 @@ from transformers.utils import CONFIG_NAME as HF_CONFIG_NAME
from vllm import envs
from vllm.logger import init_logger
from vllm.transformers_utils.repo_utils import is_mistral_model_repo
from vllm.transformers_utils.utils import parse_safetensors_file_metadata
from .config_parser_base import ConfigParserBase
@@ -49,7 +50,6 @@ except ImportError:
ALLOWED_LAYER_TYPES as ALLOWED_ATTENTION_LAYER_TYPES,
)
if envs.VLLM_USE_MODELSCOPE:
from modelscope import AutoConfig
else:
@@ -581,7 +581,11 @@ def get_config(
try:
# First check for Mistral to avoid defaulting to
# Transformers implementation.
if file_or_path_exists(model, MISTRAL_CONFIG_NAME, revision=revision):
if is_mistral_model_repo(
model_name_or_path=str(model), revision=revision
) and file_or_path_exists(
model=model, config_name=MISTRAL_CONFIG_NAME, revision=revision
):
config_format = "mistral"
elif (_is_gguf and not _is_remote_gguf) or file_or_path_exists(
model, HF_CONFIG_NAME, revision=revision
@@ -1,9 +1,12 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Iterator
from contextlib import contextmanager
from typing import final
import torch
from huggingface_hub import constants
from safetensors.torch import _TYPES as _SAFETENSORS_TO_TORCH_DTYPE
from transformers import PretrainedConfig
@@ -14,6 +17,7 @@ from vllm.config.model_arch import (
from vllm.config.utils import getattr_iter
from vllm.logger import init_logger
from vllm.transformers_utils.config import (
ConfigFormat,
try_get_safetensors_metadata,
)
from vllm.utils.torch_utils import common_broadcastable_dtype
@@ -21,6 +25,22 @@ from vllm.utils.torch_utils import common_broadcastable_dtype
logger = init_logger(__name__)
@contextmanager
def _maybe_patch_hf_hub_constants(config_format: ConfigFormat) -> Iterator[None]:
if config_format == "mistral":
hf_safetensors_single_file = constants.SAFETENSORS_SINGLE_FILE
hf_safetensors_index_file = constants.SAFETENSORS_INDEX_FILE
constants.SAFETENSORS_SINGLE_FILE = "consolidated.safetensors"
constants.SAFETENSORS_INDEX_FILE = "consolidated.safetensors.index.json"
try:
yield
finally:
constants.SAFETENSORS_SINGLE_FILE = hf_safetensors_single_file
constants.SAFETENSORS_INDEX_FILE = hf_safetensors_index_file
else:
yield
class ModelArchConfigConvertorBase:
def __init__(self, hf_config: PretrainedConfig, hf_text_config: PretrainedConfig):
self.hf_config = hf_config
@@ -123,7 +143,11 @@ class ModelArchConfigConvertorBase:
@final
@classmethod
def get_torch_dtype(
cls, hf_config: PretrainedConfig, model_id: str, revision: str | None
cls,
hf_config: PretrainedConfig,
model_id: str,
revision: str | None,
config_format: ConfigFormat,
):
# NOTE: getattr(config, "dtype", torch.float32) is not correct
# because config.dtype can be None.
@@ -140,7 +164,8 @@ class ModelArchConfigConvertorBase:
# Try to read the dtype of the weights if they are in safetensors format
if config_dtype is None:
repo_mt = try_get_safetensors_metadata(model_id, revision=revision)
with _maybe_patch_hf_hub_constants(config_format):
repo_mt = try_get_safetensors_metadata(model_id, revision=revision)
if repo_mt and (files_mt := repo_mt.files_metadata):
param_dtypes: set[torch.dtype] = {
+36
View File
@@ -127,6 +127,42 @@ def list_filtered_repo_files(
return file_list
def any_pattern_in_repo_files(
model_name_or_path: str,
allow_patterns: list[str],
revision: str | None = None,
repo_type: str | None = None,
token: str | bool | None = None,
):
return (
len(
list_filtered_repo_files(
model_name_or_path=model_name_or_path,
allow_patterns=allow_patterns,
revision=revision,
repo_type=repo_type,
token=token,
)
)
> 0
)
def is_mistral_model_repo(
model_name_or_path: str,
revision: str | None = None,
repo_type: str | None = None,
token: str | bool | None = None,
) -> bool:
return any_pattern_in_repo_files(
model_name_or_path=model_name_or_path,
allow_patterns=["consolidated*.safetensors"],
revision=revision,
repo_type=repo_type,
token=token,
)
def file_exists(
repo_id: str,
file_name: str,