Default model load/config/tokenizer to mistral format if relevant files exist (#28659)
Signed-off-by: Julien Denize <julien.denize@mistral.ai> Signed-off-by: Julien Denize <40604584+juliendenize@users.noreply.github.com> Signed-off-by: mgoin <mgoin64@gmail.com> Signed-off-by: Michael Goin <mgoin64@gmail.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Co-authored-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
co-authored by
gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
mgoin
parent
c68c7b403d
commit
57430fc95c
@@ -208,7 +208,7 @@ def test_mistral_format(
|
||||
with vllm_runner(
|
||||
model,
|
||||
dtype=dtype,
|
||||
tokenizer_mode="auto",
|
||||
tokenizer_mode="hf",
|
||||
load_format="safetensors",
|
||||
config_format="hf",
|
||||
) as hf_format_model:
|
||||
|
||||
@@ -50,12 +50,24 @@ def test_hf_model_weights_mapper(model_arch: str):
|
||||
model_info.check_available_online(on_fail="skip")
|
||||
model_info.check_transformers_version(on_fail="skip")
|
||||
|
||||
is_mistral_model = model_arch in [
|
||||
"Mistral3ForConditionalGeneration",
|
||||
"PixtralForConditionalGeneration",
|
||||
"VoxtralForConditionalGeneration",
|
||||
]
|
||||
|
||||
if not is_mistral_model or model_info.tokenizer_mode == "mistral":
|
||||
tokenizer_mode = model_info.tokenizer_mode
|
||||
else:
|
||||
tokenizer_mode = "hf"
|
||||
|
||||
model_id = model_info.default
|
||||
|
||||
model_config = ModelConfig(
|
||||
model_id,
|
||||
tokenizer=model_info.tokenizer or model_id,
|
||||
tokenizer_mode=model_info.tokenizer_mode,
|
||||
tokenizer_mode=tokenizer_mode,
|
||||
config_format="hf",
|
||||
revision=model_info.revision,
|
||||
trust_remote_code=model_info.trust_remote_code,
|
||||
hf_overrides=model_info.hf_overrides,
|
||||
|
||||
@@ -259,6 +259,9 @@ def validate_generated_texts(
|
||||
tensor_parallel_size=vllm_tp_size,
|
||||
enforce_eager=False,
|
||||
default_torch_num_threads=1,
|
||||
tokenizer_mode="hf",
|
||||
load_format="hf",
|
||||
config_format="hf",
|
||||
) as llm:
|
||||
vllm_outputs = llm.generate_greedy(prompts, max_tokens)
|
||||
vllm_logs = log_generated_texts(prompts, vllm_outputs, "VllmRunner")
|
||||
|
||||
@@ -128,6 +128,12 @@ CONFIGS: dict[str, ServerConfig] = {
|
||||
"arguments": [
|
||||
"--enforce-eager",
|
||||
"--no-enable-prefix-caching",
|
||||
"--tokenizer_mode",
|
||||
"hf",
|
||||
"--load_format",
|
||||
"hf",
|
||||
"--config_format",
|
||||
"hf",
|
||||
"--tool-call-parser",
|
||||
"mistral",
|
||||
"--chat-template",
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, call, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.transformers_utils.config import list_filtered_repo_files
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"allow_patterns,expected_relative_files",
|
||||
[
|
||||
(
|
||||
["*.json", "correct*.txt"],
|
||||
["json_file.json", "subfolder/correct.txt", "correct_2.txt"],
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_list_filtered_repo_files(
|
||||
allow_patterns: list[str], expected_relative_files: list[str]
|
||||
):
|
||||
with tempfile.TemporaryDirectory() as tmp_dir:
|
||||
# Prep folder and files
|
||||
path_tmp_dir = Path(tmp_dir)
|
||||
subfolder = path_tmp_dir / "subfolder"
|
||||
subfolder.mkdir()
|
||||
(path_tmp_dir / "json_file.json").touch()
|
||||
(path_tmp_dir / "correct_2.txt").touch()
|
||||
(path_tmp_dir / "uncorrect.txt").touch()
|
||||
(path_tmp_dir / "uncorrect.jpeg").touch()
|
||||
(subfolder / "correct.txt").touch()
|
||||
(subfolder / "uncorrect_sub.txt").touch()
|
||||
|
||||
def _glob_path() -> list[str]:
|
||||
return [
|
||||
str(file.relative_to(path_tmp_dir))
|
||||
for file in path_tmp_dir.glob("**/*")
|
||||
if file.is_file()
|
||||
]
|
||||
|
||||
# Patch list_repo_files called by fn
|
||||
with patch(
|
||||
"vllm.transformers_utils.config.list_repo_files",
|
||||
MagicMock(return_value=_glob_path()),
|
||||
) as mock_list_repo_files:
|
||||
out_files = sorted(
|
||||
list_filtered_repo_files(
|
||||
tmp_dir, allow_patterns, "revision", "model", "token"
|
||||
)
|
||||
)
|
||||
assert out_files == sorted(expected_relative_files)
|
||||
assert mock_list_repo_files.call_count == 1
|
||||
assert mock_list_repo_files.call_args_list[0] == call(
|
||||
repo_id=tmp_dir,
|
||||
revision="revision",
|
||||
repo_type="model",
|
||||
token="token",
|
||||
)
|
||||
@@ -2,7 +2,11 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
|
||||
from vllm.transformers_utils.utils import is_cloud_storage, is_gcs, is_s3
|
||||
from vllm.transformers_utils.utils import (
|
||||
is_cloud_storage,
|
||||
is_gcs,
|
||||
is_s3,
|
||||
)
|
||||
|
||||
|
||||
def test_is_gcs():
|
||||
|
||||
@@ -46,11 +46,15 @@ EAGLE_SPEC_CONFIG = {
|
||||
|
||||
PARAMS_MODELS_BACKENDS_TOKENIZER_MODE = [
|
||||
("mistralai/Ministral-8B-Instruct-2410", "xgrammar", "auto", None),
|
||||
("mistralai/Ministral-8B-Instruct-2410", "guidance", "auto", None),
|
||||
# FIXME: Since "auto" will use Mistral tokenizer and these backends do not support
|
||||
# it, we skip these tests for now.
|
||||
# ("mistralai/Ministral-8B-Instruct-2410", "guidance", "auto", None),
|
||||
# ("mistralai/Ministral-8B-Instruct-2410", "lm-format-enforcer", "auto", None),
|
||||
("mistralai/Ministral-8B-Instruct-2410", "guidance", "hf", None),
|
||||
pytest.param(
|
||||
"mistralai/Ministral-8B-Instruct-2410",
|
||||
"lm-format-enforcer",
|
||||
"auto",
|
||||
"hf",
|
||||
None,
|
||||
marks=pytest.mark.skip(
|
||||
reason=(
|
||||
@@ -80,7 +84,7 @@ PARAMS_MODELS_BACKENDS_TOKENIZER_MODE = [
|
||||
# ("mistralai/Ministral-8B-Instruct-2410", "outlines", "mistral", None),
|
||||
# ("Qwen/Qwen2.5-1.5B-Instruct", "guidance", "auto"),
|
||||
("mistralai/Ministral-8B-Instruct-2410", "outlines", "auto", NGRAM_SPEC_CONFIG),
|
||||
("mistralai/Ministral-8B-Instruct-2410", "guidance", "auto", NGRAM_SPEC_CONFIG),
|
||||
("mistralai/Ministral-8B-Instruct-2410", "guidance", "hf", NGRAM_SPEC_CONFIG),
|
||||
("Qwen/Qwen2.5-1.5B-Instruct", "xgrammar", "auto", NGRAM_SPEC_CONFIG),
|
||||
("meta-llama/Meta-Llama-3.1-8B-Instruct", "xgrammar", "auto", EAGLE_SPEC_CONFIG),
|
||||
]
|
||||
@@ -151,6 +155,8 @@ def test_structured_output(
|
||||
),
|
||||
seed=120,
|
||||
tokenizer_mode=tokenizer_mode,
|
||||
load_format="auto" if not model_name.startswith("mistralai/") else "hf",
|
||||
config_format="auto" if not model_name.startswith("mistralai/") else "hf",
|
||||
speculative_config=speculative_config,
|
||||
)
|
||||
|
||||
@@ -720,6 +726,8 @@ def test_structured_output_auto_mode(
|
||||
max_model_len=1024,
|
||||
structured_outputs_config=dict(backend="auto"),
|
||||
tokenizer_mode=tokenizer_mode,
|
||||
load_format="auto",
|
||||
config_format="auto",
|
||||
)
|
||||
|
||||
sampling_params = SamplingParams(
|
||||
|
||||
Reference in New Issue
Block a user