[Bugfix] Fix offline mode when using mistral_common (#9457)
This commit is contained in:
@@ -1,50 +1,56 @@
|
||||
"""Tests for HF_HUB_OFFLINE mode"""
|
||||
import importlib
|
||||
import sys
|
||||
import weakref
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm import LLM
|
||||
from vllm.distributed import cleanup_dist_env_and_memory
|
||||
|
||||
MODEL_NAME = "facebook/opt-125m"
|
||||
MODEL_CONFIGS = [
|
||||
{
|
||||
"model": "facebook/opt-125m",
|
||||
"enforce_eager": True,
|
||||
"gpu_memory_utilization": 0.20,
|
||||
"max_model_len": 64,
|
||||
"max_num_batched_tokens": 64,
|
||||
"max_num_seqs": 64,
|
||||
"tensor_parallel_size": 1,
|
||||
},
|
||||
{
|
||||
"model": "mistralai/Mistral-7B-Instruct-v0.1",
|
||||
"enforce_eager": True,
|
||||
"gpu_memory_utilization": 0.95,
|
||||
"max_model_len": 64,
|
||||
"max_num_batched_tokens": 64,
|
||||
"max_num_seqs": 64,
|
||||
"tensor_parallel_size": 1,
|
||||
"tokenizer_mode": "mistral",
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def llm():
|
||||
# pytest caches the fixture so we use weakref.proxy to
|
||||
# enable garbage collection
|
||||
llm = LLM(model=MODEL_NAME,
|
||||
max_num_batched_tokens=4096,
|
||||
tensor_parallel_size=1,
|
||||
gpu_memory_utilization=0.10,
|
||||
enforce_eager=True)
|
||||
def cache_models():
|
||||
# Cache model files first
|
||||
for model_config in MODEL_CONFIGS:
|
||||
LLM(**model_config)
|
||||
cleanup_dist_env_and_memory()
|
||||
|
||||
with llm.deprecate_legacy_api():
|
||||
yield weakref.proxy(llm)
|
||||
|
||||
del llm
|
||||
|
||||
cleanup_dist_env_and_memory()
|
||||
yield
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_offline_mode(llm: LLM, monkeypatch):
|
||||
# we use the llm fixture to ensure the model files are in-cache
|
||||
del llm
|
||||
|
||||
@pytest.mark.usefixtures("cache_models")
|
||||
def test_offline_mode(monkeypatch):
|
||||
# Set HF to offline mode and ensure we can still construct an LLM
|
||||
try:
|
||||
monkeypatch.setenv("HF_HUB_OFFLINE", "1")
|
||||
# Need to re-import huggingface_hub and friends to setup offline mode
|
||||
_re_import_modules()
|
||||
# Cached model files should be used in offline mode
|
||||
LLM(model=MODEL_NAME,
|
||||
max_num_batched_tokens=4096,
|
||||
tensor_parallel_size=1,
|
||||
gpu_memory_utilization=0.20,
|
||||
enforce_eager=True)
|
||||
for model_config in MODEL_CONFIGS:
|
||||
LLM(**model_config)
|
||||
finally:
|
||||
# Reset the environment after the test
|
||||
# NB: Assuming tests are run in online mode
|
||||
|
||||
Reference in New Issue
Block a user