Default model load/config/tokenizer to mistral format if relevant files exist (#28659)
Signed-off-by: Julien Denize <julien.denize@mistral.ai> Signed-off-by: Julien Denize <40604584+juliendenize@users.noreply.github.com> Signed-off-by: mgoin <mgoin64@gmail.com> Signed-off-by: Michael Goin <mgoin64@gmail.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Co-authored-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
co-authored by
gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
mgoin
parent
c68c7b403d
commit
57430fc95c
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import fnmatch
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
@@ -355,6 +356,41 @@ def list_repo_files(
|
||||
return with_retry(lookup_files, "Error retrieving file list")
|
||||
|
||||
|
||||
def list_filtered_repo_files(
|
||||
model_name_or_path: str,
|
||||
allow_patterns: list[str],
|
||||
revision: str | None = None,
|
||||
repo_type: str | None = None,
|
||||
token: str | bool | None = None,
|
||||
) -> list[str]:
|
||||
try:
|
||||
all_files = list_repo_files(
|
||||
repo_id=model_name_or_path,
|
||||
revision=revision,
|
||||
token=token,
|
||||
repo_type=repo_type,
|
||||
)
|
||||
except Exception:
|
||||
logger.error(
|
||||
"Error retrieving file list. Please ensure your `model_name_or_path`"
|
||||
"`repo_type`, `token` and `revision` arguments are correctly set. "
|
||||
"Returning an empty list."
|
||||
)
|
||||
return []
|
||||
|
||||
file_list = []
|
||||
# Filter patterns on filenames
|
||||
for pattern in allow_patterns:
|
||||
file_list.extend(
|
||||
[
|
||||
file
|
||||
for file in all_files
|
||||
if fnmatch.fnmatch(os.path.basename(file), pattern)
|
||||
]
|
||||
)
|
||||
return file_list
|
||||
|
||||
|
||||
def file_exists(
|
||||
repo_id: str,
|
||||
file_name: str,
|
||||
@@ -619,10 +655,14 @@ def get_config(
|
||||
|
||||
if config_format == "auto":
|
||||
try:
|
||||
if is_gguf or file_or_path_exists(model, HF_CONFIG_NAME, revision=revision):
|
||||
config_format = "hf"
|
||||
elif file_or_path_exists(model, MISTRAL_CONFIG_NAME, revision=revision):
|
||||
# First check for Mistral to avoid defaulting to
|
||||
# Transformers implementation.
|
||||
if file_or_path_exists(model, MISTRAL_CONFIG_NAME, revision=revision):
|
||||
config_format = "mistral"
|
||||
elif is_gguf or file_or_path_exists(
|
||||
model, HF_CONFIG_NAME, revision=revision
|
||||
):
|
||||
config_format = "hf"
|
||||
else:
|
||||
raise ValueError(
|
||||
"Could not detect config format for no config file found. "
|
||||
|
||||
@@ -118,7 +118,7 @@ def _remap_general_mistral_args(config: dict) -> dict:
|
||||
"model_type": ("model_type", "transformer"),
|
||||
"hidden_act": ("activation", "silu"),
|
||||
"tie_word_embeddings": ("tied_embeddings", False),
|
||||
"max_seq_len": ("max_seq_len", 128_000),
|
||||
"max_seq_len": ("max_seq_len", config.get("max_position_embeddings", 128_000)),
|
||||
"max_position_embeddings": ("max_position_embeddings", 128_000),
|
||||
}
|
||||
|
||||
|
||||
@@ -3,8 +3,8 @@
|
||||
|
||||
import contextlib
|
||||
import copy
|
||||
import importlib.util
|
||||
import os
|
||||
import warnings
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Any, TypeAlias
|
||||
@@ -15,7 +15,10 @@ from typing_extensions import assert_never
|
||||
|
||||
from vllm import envs
|
||||
from vllm.logger import init_logger
|
||||
from vllm.transformers_utils.config import get_sentence_transformer_tokenizer_config
|
||||
from vllm.transformers_utils.config import (
|
||||
get_sentence_transformer_tokenizer_config,
|
||||
list_filtered_repo_files,
|
||||
)
|
||||
from vllm.transformers_utils.tokenizers import MistralTokenizer
|
||||
from vllm.transformers_utils.utils import check_gguf_file
|
||||
|
||||
@@ -182,25 +185,29 @@ def get_tokenizer(
|
||||
kwargs["gguf_file"] = Path(tokenizer_name).name
|
||||
tokenizer_name = Path(tokenizer_name).parent
|
||||
|
||||
# if tokenizer is from official mistral org
|
||||
is_from_mistral_org = str(tokenizer_name).split("/")[0] == "mistralai"
|
||||
if is_from_mistral_org and tokenizer_mode != "mistral":
|
||||
warnings.warn(
|
||||
"It is strongly recommended to run mistral models with "
|
||||
'`--tokenizer-mode "mistral"` to ensure correct '
|
||||
"encoding and decoding.",
|
||||
FutureWarning,
|
||||
stacklevel=2,
|
||||
# if `tokenizer_mode` == "auto", check if tokenizer can be loaded via Mistral format
|
||||
# first to use official Mistral tokenizer if possible.
|
||||
mistral_common_installed = importlib.util.find_spec("mistral_common") is not None
|
||||
if tokenizer_mode == "auto" and mistral_common_installed:
|
||||
allow_patterns = ["tekken.json", "tokenizer.model.v*"]
|
||||
files_list = list_filtered_repo_files(
|
||||
model_name_or_path=str(tokenizer_name),
|
||||
allow_patterns=allow_patterns,
|
||||
revision=revision,
|
||||
)
|
||||
if len(files_list) > 0:
|
||||
tokenizer_mode = "mistral"
|
||||
|
||||
tokenizer: AnyTokenizer
|
||||
if tokenizer_mode == "mistral":
|
||||
logger.debug_once(f"Loading MistralTokenizer from {tokenizer_name}")
|
||||
tokenizer = MistralTokenizer.from_pretrained(
|
||||
str(tokenizer_name), revision=revision
|
||||
)
|
||||
elif tokenizer_mode == "custom":
|
||||
from vllm.transformers_utils.tokenizer_base import TokenizerRegistry
|
||||
|
||||
logger.debug_once(f"Loading CustomTokenizer from {tokenizer_name}")
|
||||
tokenizer = TokenizerRegistry.get_tokenizer(
|
||||
str(tokenizer_name),
|
||||
*args,
|
||||
@@ -210,6 +217,7 @@ def get_tokenizer(
|
||||
)
|
||||
else:
|
||||
try:
|
||||
logger.debug_once(f"Loading AutoTokenizer from {tokenizer_name}")
|
||||
tokenizer = AutoTokenizer.from_pretrained(
|
||||
tokenizer_name,
|
||||
*args,
|
||||
|
||||
Reference in New Issue
Block a user