[CI/Build] vLLM cache directory for images (#6444)
This commit is contained in:
@@ -0,0 +1,11 @@
|
||||
from pathlib import Path
|
||||
|
||||
import vllm.envs as envs
|
||||
|
||||
|
||||
def get_cache_dir():
|
||||
"""Get the path to the cache for storing downloaded assets."""
|
||||
path = Path(envs.VLLM_ASSETS_CACHE)
|
||||
path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
return path
|
||||
@@ -0,0 +1,47 @@
|
||||
import shutil
|
||||
from dataclasses import dataclass
|
||||
from functools import cached_property, lru_cache
|
||||
from typing import Literal
|
||||
|
||||
import requests
|
||||
from PIL import Image
|
||||
|
||||
from vllm.multimodal.utils import fetch_image
|
||||
|
||||
from .base import get_cache_dir
|
||||
|
||||
|
||||
@lru_cache
|
||||
def get_air_example_data_2_asset(filename: str) -> Image.Image:
|
||||
"""
|
||||
Download and open an image from
|
||||
``s3://air-example-data-2/vllm_opensource_llava/``.
|
||||
"""
|
||||
image_directory = get_cache_dir() / "air-example-data-2"
|
||||
image_directory.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
image_path = image_directory / filename
|
||||
if not image_path.exists():
|
||||
base_url = "https://air-example-data-2.s3.us-west-2.amazonaws.com/vllm_opensource_llava"
|
||||
|
||||
with requests.get(f"{base_url}/{filename}", stream=True) as response:
|
||||
response.raise_for_status()
|
||||
|
||||
with image_path.open("wb") as f:
|
||||
shutil.copyfileobj(response.raw, f)
|
||||
|
||||
return Image.open(image_path)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ImageAsset:
|
||||
name: Literal["stop_sign", "cherry_blossom", "boardwalk"]
|
||||
|
||||
@cached_property
|
||||
def pil_image(self) -> Image.Image:
|
||||
if self.name == "boardwalk":
|
||||
return fetch_image(
|
||||
"https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg"
|
||||
)
|
||||
|
||||
return get_air_example_data_2_asset(f"{self.name}.jpg")
|
||||
@@ -189,10 +189,10 @@ def gpu_p2p_access_check(src: int, tgt: int) -> bool:
|
||||
cuda_visible_devices = envs.CUDA_VISIBLE_DEVICES
|
||||
if cuda_visible_devices is None:
|
||||
cuda_visible_devices = ",".join(str(i) for i in range(num_dev))
|
||||
VLLM_CONFIG_ROOT = envs.VLLM_CONFIG_ROOT
|
||||
path = os.path.expanduser(
|
||||
f"{VLLM_CONFIG_ROOT}/vllm/gpu_p2p_access_cache_for_{cuda_visible_devices}.json"
|
||||
)
|
||||
|
||||
path = os.path.join(
|
||||
envs.VLLM_CACHE_ROOT,
|
||||
f"gpu_p2p_access_cache_for_{cuda_visible_devices}.json")
|
||||
os.makedirs(os.path.dirname(path), exist_ok=True)
|
||||
from vllm.distributed.parallel_state import get_world_group
|
||||
if ((not is_distributed or get_world_group().local_rank == 0)
|
||||
|
||||
+48
-6
@@ -17,7 +17,8 @@ if TYPE_CHECKING:
|
||||
S3_ACCESS_KEY_ID: Optional[str] = None
|
||||
S3_SECRET_ACCESS_KEY: Optional[str] = None
|
||||
S3_ENDPOINT_URL: Optional[str] = None
|
||||
VLLM_CONFIG_ROOT: str = ""
|
||||
VLLM_CACHE_ROOT: str = os.path.expanduser("~/.cache/vllm")
|
||||
VLLM_CONFIG_ROOT: str = os.path.expanduser("~/.config/vllm")
|
||||
VLLM_USAGE_STATS_SERVER: str = "https://stats.vllm.ai"
|
||||
VLLM_NO_USAGE_STATS: bool = False
|
||||
VLLM_DO_NOT_TRACK: bool = False
|
||||
@@ -31,10 +32,11 @@ if TYPE_CHECKING:
|
||||
VLLM_OPENVINO_KVCACHE_SPACE: int = 0
|
||||
VLLM_OPENVINO_CPU_KV_CACHE_PRECISION: Optional[str] = None
|
||||
VLLM_OPENVINO_ENABLE_QUANTIZED_WEIGHTS: bool = False
|
||||
VLLM_XLA_CACHE_PATH: str = "~/.vllm/xla_cache/"
|
||||
VLLM_XLA_CACHE_PATH: str = os.path.join(VLLM_CACHE_ROOT, "xla_cache")
|
||||
VLLM_FUSED_MOE_CHUNK_SIZE: int = 64 * 1024
|
||||
VLLM_USE_RAY_COMPILED_DAG: bool = False
|
||||
VLLM_WORKER_MULTIPROC_METHOD: str = "fork"
|
||||
VLLM_ASSETS_CACHE: str = os.path.join(VLLM_CACHE_ROOT, "assets")
|
||||
VLLM_IMAGE_FETCH_TIMEOUT: int = 5
|
||||
VLLM_TARGET_DEVICE: str = "cuda"
|
||||
MAX_JOBS: Optional[str] = None
|
||||
@@ -45,6 +47,21 @@ if TYPE_CHECKING:
|
||||
CMAKE_BUILD_TYPE: Optional[str] = None
|
||||
VERBOSE: bool = False
|
||||
|
||||
|
||||
def get_default_cache_root():
|
||||
return os.getenv(
|
||||
"XDG_CACHE_HOME",
|
||||
os.path.join(os.path.expanduser("~"), ".cache"),
|
||||
)
|
||||
|
||||
|
||||
def get_default_config_root():
|
||||
return os.getenv(
|
||||
"XDG_CONFIG_HOME",
|
||||
os.path.join(os.path.expanduser("~"), ".config"),
|
||||
)
|
||||
|
||||
|
||||
# The begin-* and end* here are used by the documentation generator
|
||||
# to extract the used env vars.
|
||||
|
||||
@@ -89,15 +106,28 @@ environment_variables: Dict[str, Callable[[], Any]] = {
|
||||
lambda: bool(int(os.getenv('VERBOSE', '0'))),
|
||||
|
||||
# Root directory for VLLM configuration files
|
||||
# Defaults to `~/.config/vllm` unless `XDG_CONFIG_HOME` is set
|
||||
# Note that this not only affects how vllm finds its configuration files
|
||||
# during runtime, but also affects how vllm installs its configuration
|
||||
# files during **installation**.
|
||||
"VLLM_CONFIG_ROOT":
|
||||
lambda: os.environ.get("VLLM_CONFIG_ROOT", None) or os.getenv(
|
||||
"XDG_CONFIG_HOME", None) or os.path.expanduser("~/.config"),
|
||||
lambda: os.path.expanduser(
|
||||
os.getenv(
|
||||
"VLLM_CONFIG_ROOT",
|
||||
os.path.join(get_default_config_root(), "vllm"),
|
||||
)),
|
||||
|
||||
# ================== Runtime Env Vars ==================
|
||||
|
||||
# Root directory for VLLM cache files
|
||||
# Defaults to `~/.cache/vllm` unless `XDG_CACHE_HOME` is set
|
||||
"VLLM_CACHE_ROOT":
|
||||
lambda: os.path.expanduser(
|
||||
os.getenv(
|
||||
"VLLM_CACHE_ROOT",
|
||||
os.path.join(get_default_cache_root(), "vllm"),
|
||||
)),
|
||||
|
||||
# used in distributed environment to determine the master address
|
||||
'VLLM_HOST_IP':
|
||||
lambda: os.getenv('VLLM_HOST_IP', "") or os.getenv("HOST_IP", ""),
|
||||
@@ -242,6 +272,14 @@ environment_variables: Dict[str, Callable[[], Any]] = {
|
||||
"VLLM_WORKER_MULTIPROC_METHOD":
|
||||
lambda: os.getenv("VLLM_WORKER_MULTIPROC_METHOD", "fork"),
|
||||
|
||||
# Path to the cache for storing downloaded assets
|
||||
"VLLM_ASSETS_CACHE":
|
||||
lambda: os.path.expanduser(
|
||||
os.getenv(
|
||||
"VLLM_ASSETS_CACHE",
|
||||
os.path.join(get_default_cache_root(), "vllm", "assets"),
|
||||
)),
|
||||
|
||||
# Timeout for fetching images when serving multimodal models
|
||||
# Default is 5 seconds
|
||||
"VLLM_IMAGE_FETCH_TIMEOUT":
|
||||
@@ -250,7 +288,11 @@ environment_variables: Dict[str, Callable[[], Any]] = {
|
||||
# Path to the XLA persistent cache directory.
|
||||
# Only used for XLA devices such as TPUs.
|
||||
"VLLM_XLA_CACHE_PATH":
|
||||
lambda: os.getenv("VLLM_XLA_CACHE_PATH", "~/.vllm/xla_cache/"),
|
||||
lambda: os.path.expanduser(
|
||||
os.getenv(
|
||||
"VLLM_ASSETS_CACHE",
|
||||
os.path.join(get_default_cache_root(), "vllm", "xla_cache"),
|
||||
)),
|
||||
"VLLM_FUSED_MOE_CHUNK_SIZE":
|
||||
lambda: int(os.getenv("VLLM_FUSED_MOE_CHUNK_SIZE", "65536")),
|
||||
|
||||
@@ -262,7 +304,7 @@ environment_variables: Dict[str, Callable[[], Any]] = {
|
||||
# end-env-vars-definition
|
||||
|
||||
|
||||
def __getattr__(name):
|
||||
def __getattr__(name: str):
|
||||
# lazy evaluation of environment variables
|
||||
if name in environment_variables:
|
||||
return environment_variables[name]()
|
||||
|
||||
@@ -19,9 +19,8 @@ import vllm.envs as envs
|
||||
from vllm.version import __version__ as VLLM_VERSION
|
||||
|
||||
_config_home = envs.VLLM_CONFIG_ROOT
|
||||
_USAGE_STATS_JSON_PATH = os.path.join(_config_home, "vllm/usage_stats.json")
|
||||
_USAGE_STATS_DO_NOT_TRACK_PATH = os.path.join(_config_home,
|
||||
"vllm/do_not_track")
|
||||
_USAGE_STATS_JSON_PATH = os.path.join(_config_home, "usage_stats.json")
|
||||
_USAGE_STATS_DO_NOT_TRACK_PATH = os.path.join(_config_home, "do_not_track")
|
||||
_USAGE_STATS_ENABLED = None
|
||||
_USAGE_STATS_SERVER = envs.VLLM_USAGE_STATS_SERVER
|
||||
|
||||
|
||||
@@ -98,8 +98,7 @@ class TPUWorker(LoraNotSupportedWorkerBase):
|
||||
# Use persistent cache to avoid XLA recompilation.
|
||||
# NOTE(woosuk): This does not completely eliminate the recompilation
|
||||
# overhead because dynamo does not cache the compiled results.
|
||||
xr.initialize_cache(os.path.expanduser(envs.VLLM_XLA_CACHE_PATH),
|
||||
readonly=False)
|
||||
xr.initialize_cache(envs.VLLM_XLA_CACHE_PATH, readonly=False)
|
||||
|
||||
def load_model(self):
|
||||
self.model_runner.load_model()
|
||||
|
||||
Reference in New Issue
Block a user