diff --git a/vllm/config/load.py b/vllm/config/load.py index 93240ec5fc..07feaeb961 100644 --- a/vllm/config/load.py +++ b/vllm/config/load.py @@ -9,6 +9,9 @@ from vllm.config.utils import config from vllm.logger import init_logger from vllm.utils.hashing import safe_hash +DEFAULT_SAFETENSORS_PREFETCH_NUM_THREADS = 8 +DEFAULT_SAFETENSORS_PREFETCH_BLOCK_SIZE = 16 * 1024 * 1024 + if TYPE_CHECKING: from vllm.model_executor.model_loader import LoadFormats from vllm.model_executor.model_loader.tensorizer import TensorizerConfig @@ -79,6 +82,15 @@ class LoadConfig: was quantized using torchao and saved using safetensors. Needs `torchao >= 0.14.0`. """ + safetensors_prefetch_num_threads: int = Field( + default=DEFAULT_SAFETENSORS_PREFETCH_NUM_THREADS, ge=1 + ) + """Number of worker threads used to prefetch safetensors checkpoint files + into the OS page cache when safetensors prefetching is enabled.""" + safetensors_prefetch_block_size: int = Field( + default=DEFAULT_SAFETENSORS_PREFETCH_BLOCK_SIZE, ge=1 + ) + """Read size in bytes for each safetensors checkpoint file prefetch.""" model_loader_extra_config: dict | TensorizerConfig = Field(default_factory=dict) """Extra config for model loader. This will be passed to the model loader corresponding to the chosen load_format.""" diff --git a/vllm/engine/arg_utils.py b/vllm/engine/arg_utils.py index 4b0e2f61e6..e65ef6e425 100644 --- a/vllm/engine/arg_utils.py +++ b/vllm/engine/arg_utils.py @@ -355,7 +355,11 @@ def _compute_kwargs(cls: ConfigType) -> dict[str, dict[str, Any]]: if name == "max_model_len": kwargs[name]["type"] = human_readable_int_or_auto kwargs[name]["help"] += f"\n\n{human_readable_int_or_auto.__doc__}" - elif name in ("max_num_batched_tokens", "kv_cache_memory_bytes"): + elif name in ( + "max_num_batched_tokens", + "kv_cache_memory_bytes", + "safetensors_prefetch_block_size", + ): kwargs[name]["type"] = human_readable_int kwargs[name]["help"] += f"\n\n{human_readable_int.__doc__}" else: @@ -424,6 +428,8 @@ class EngineArgs: allowed_media_domains: list[str] | None = ModelConfig.allowed_media_domains download_dir: str | None = LoadConfig.download_dir safetensors_load_strategy: str | None = LoadConfig.safetensors_load_strategy + safetensors_prefetch_num_threads: int = LoadConfig.safetensors_prefetch_num_threads + safetensors_prefetch_block_size: int = LoadConfig.safetensors_prefetch_block_size load_format: str | LoadFormats = LoadConfig.load_format config_format: str = ModelConfig.config_format dtype: ModelDType = ModelConfig.dtype @@ -844,6 +850,14 @@ class EngineArgs: load_group.add_argument( "--safetensors-load-strategy", **load_kwargs["safetensors_load_strategy"] ) + load_group.add_argument( + "--safetensors-prefetch-num-threads", + **load_kwargs["safetensors_prefetch_num_threads"], + ) + load_group.add_argument( + "--safetensors-prefetch-block-size", + **load_kwargs["safetensors_prefetch_block_size"], + ) load_group.add_argument( "--model-loader-extra-config", **load_kwargs["model_loader_extra_config"] ) @@ -1584,6 +1598,8 @@ class EngineArgs: load_format=self.load_format, download_dir=self.download_dir, safetensors_load_strategy=self.safetensors_load_strategy, + safetensors_prefetch_num_threads=self.safetensors_prefetch_num_threads, + safetensors_prefetch_block_size=self.safetensors_prefetch_block_size, model_loader_extra_config=self.model_loader_extra_config, ignore_patterns=self.ignore_patterns, use_tqdm_on_load=self.use_tqdm_on_load, diff --git a/vllm/model_executor/model_loader/default_loader.py b/vllm/model_executor/model_loader/default_loader.py index a760920286..5b5632e319 100644 --- a/vllm/model_executor/model_loader/default_loader.py +++ b/vllm/model_executor/model_loader/default_loader.py @@ -256,6 +256,12 @@ class DefaultModelLoader(BaseModelLoader): self.load_config.use_tqdm_on_load, self.load_config.safetensors_load_strategy, local_expert_ids=self.local_expert_ids, + safetensors_prefetch_num_threads=( + self.load_config.safetensors_prefetch_num_threads + ), + safetensors_prefetch_block_size=( + self.load_config.safetensors_prefetch_block_size + ), ) else: if extra_config.get("enable_multithread_load"): diff --git a/vllm/model_executor/model_loader/weight_utils.py b/vllm/model_executor/model_loader/weight_utils.py index 31b00df4e4..c418c29a8c 100644 --- a/vllm/model_executor/model_loader/weight_utils.py +++ b/vllm/model_executor/model_loader/weight_utils.py @@ -30,7 +30,11 @@ from transformers.utils import SAFE_WEIGHTS_INDEX_NAME from vllm import envs from vllm.config import ModelConfig -from vllm.config.load import LoadConfig +from vllm.config.load import ( + DEFAULT_SAFETENSORS_PREFETCH_BLOCK_SIZE, + DEFAULT_SAFETENSORS_PREFETCH_NUM_THREADS, + LoadConfig, +) from vllm.distributed import get_tensor_model_parallel_rank, get_world_group from vllm.logger import init_logger from vllm.model_executor.layers.quantization import ( @@ -810,40 +814,57 @@ def _get_fs_type(files: list[str]) -> str: return "" -def _prefetch_checkpoint(file_path: str) -> None: +def _prefetch_checkpoint( + file_path: str, + block_size: int = DEFAULT_SAFETENSORS_PREFETCH_BLOCK_SIZE, +) -> None: """Prefetch a checkpoint file into the OS page cache. - Reads the file in 16MB blocks so the kernel caches its pages before - workers load the same file. + Reads the file in blocks so the kernel caches its pages before workers load + the same file. """ - block_size = 16 * 1024 * 1024 # 16MB + if block_size < 1: + raise ValueError("safetensors prefetch block size must be >= 1") + with open(file_path, "rb") as f: while f.read(block_size): pass -def _prefetch_all_checkpoints(sorted_files: list[str]) -> None: +def _prefetch_all_checkpoints( + sorted_files: list[str], + num_prefetch_threads: int = DEFAULT_SAFETENSORS_PREFETCH_NUM_THREADS, + block_size: int = DEFAULT_SAFETENSORS_PREFETCH_BLOCK_SIZE, +) -> None: """Start prefetching checkpoint files into page cache in a background thread.""" + if num_prefetch_threads < 1: + raise ValueError("safetensors prefetch num threads must be >= 1") + if block_size < 1: + raise ValueError("safetensors prefetch block size must be >= 1") + if torch.distributed.is_initialized(): rank = torch.distributed.get_rank() world_size = torch.distributed.get_world_size() else: rank = 0 world_size = 1 - num_prefetch_threads = 8 paths_to_prefetch = sorted_files[rank::world_size] total_for_rank = len(paths_to_prefetch) async def _prefetch_all() -> None: - semaphore = asyncio.Semaphore(num_prefetch_threads) + loop = asyncio.get_running_loop() completed = 0 next_log_pct = 10 - async def prefetch_one(path: str) -> None: + async def prefetch_one( + path: str, + executor: concurrent.futures.ThreadPoolExecutor, + ) -> None: nonlocal completed, next_log_pct try: - async with semaphore: - await asyncio.to_thread(_prefetch_checkpoint, path) + await loop.run_in_executor( + executor, _prefetch_checkpoint, path, block_size + ) completed += 1 if total_for_rank > 0 and next_log_pct <= 100: pct = 100 * completed / total_for_rank @@ -860,7 +881,12 @@ def _prefetch_all_checkpoints(sorted_files: list[str]) -> None: "Failed to prefetch checkpoint file %r.", path, exc_info=True ) - await asyncio.gather(*(prefetch_one(p) for p in paths_to_prefetch)) + with concurrent.futures.ThreadPoolExecutor( + max_workers=num_prefetch_threads + ) as executor: + await asyncio.gather( + *(prefetch_one(p, executor) for p in paths_to_prefetch) + ) def _run_prefetch() -> None: start = time.perf_counter() @@ -871,7 +897,12 @@ def _prefetch_all_checkpoints(sorted_files: list[str]) -> None: elapsed, ) - logger.info("Prefetching checkpoint files into page cache started (in background)") + logger.info( + "Prefetching checkpoint files into page cache started " + "(in background, num_threads=%d, block_size=%d bytes)", + num_prefetch_threads, + block_size, + ) threading.Thread(target=_run_prefetch, daemon=True).start() @@ -880,6 +911,9 @@ def safetensors_weights_iterator( use_tqdm_on_load: bool, safetensors_load_strategy: str | None = None, local_expert_ids: set[int] | None = None, + *, + safetensors_prefetch_num_threads: int = DEFAULT_SAFETENSORS_PREFETCH_NUM_THREADS, + safetensors_prefetch_block_size: int = DEFAULT_SAFETENSORS_PREFETCH_BLOCK_SIZE, ) -> Generator[tuple[str, torch.Tensor], None, None]: """Iterate over the weights in the model safetensor files. @@ -951,7 +985,11 @@ def safetensors_weights_iterator( ) if should_prefetch: - _prefetch_all_checkpoints(sorted_files) + _prefetch_all_checkpoints( + sorted_files, + num_prefetch_threads=safetensors_prefetch_num_threads, + block_size=safetensors_prefetch_block_size, + ) leftover_state_dict: dict[str, torch.Tensor] = {} for st_file in tqdm(