[Performance] Make safetensors checkpoint prefetch settings configurable (#41499)

Signed-off-by: Mohammad Miadh Angkad <MAngkad.BSDSBA2027@aim.edu>
This commit is contained in:
Mohammad Miadh Angkad
2026-05-10 15:55:15 +00:00
committed by GitHub
parent f396bee56f
commit 21943d4c25
4 changed files with 87 additions and 15 deletions
+12
View File
@@ -9,6 +9,9 @@ from vllm.config.utils import config
from vllm.logger import init_logger
from vllm.utils.hashing import safe_hash
DEFAULT_SAFETENSORS_PREFETCH_NUM_THREADS = 8
DEFAULT_SAFETENSORS_PREFETCH_BLOCK_SIZE = 16 * 1024 * 1024
if TYPE_CHECKING:
from vllm.model_executor.model_loader import LoadFormats
from vllm.model_executor.model_loader.tensorizer import TensorizerConfig
@@ -79,6 +82,15 @@ class LoadConfig:
was quantized using torchao and saved using safetensors.
Needs `torchao >= 0.14.0`.
"""
safetensors_prefetch_num_threads: int = Field(
default=DEFAULT_SAFETENSORS_PREFETCH_NUM_THREADS, ge=1
)
"""Number of worker threads used to prefetch safetensors checkpoint files
into the OS page cache when safetensors prefetching is enabled."""
safetensors_prefetch_block_size: int = Field(
default=DEFAULT_SAFETENSORS_PREFETCH_BLOCK_SIZE, ge=1
)
"""Read size in bytes for each safetensors checkpoint file prefetch."""
model_loader_extra_config: dict | TensorizerConfig = Field(default_factory=dict)
"""Extra config for model loader. This will be passed to the model loader
corresponding to the chosen load_format."""
+17 -1
View File
@@ -355,7 +355,11 @@ def _compute_kwargs(cls: ConfigType) -> dict[str, dict[str, Any]]:
if name == "max_model_len":
kwargs[name]["type"] = human_readable_int_or_auto
kwargs[name]["help"] += f"\n\n{human_readable_int_or_auto.__doc__}"
elif name in ("max_num_batched_tokens", "kv_cache_memory_bytes"):
elif name in (
"max_num_batched_tokens",
"kv_cache_memory_bytes",
"safetensors_prefetch_block_size",
):
kwargs[name]["type"] = human_readable_int
kwargs[name]["help"] += f"\n\n{human_readable_int.__doc__}"
else:
@@ -424,6 +428,8 @@ class EngineArgs:
allowed_media_domains: list[str] | None = ModelConfig.allowed_media_domains
download_dir: str | None = LoadConfig.download_dir
safetensors_load_strategy: str | None = LoadConfig.safetensors_load_strategy
safetensors_prefetch_num_threads: int = LoadConfig.safetensors_prefetch_num_threads
safetensors_prefetch_block_size: int = LoadConfig.safetensors_prefetch_block_size
load_format: str | LoadFormats = LoadConfig.load_format
config_format: str = ModelConfig.config_format
dtype: ModelDType = ModelConfig.dtype
@@ -844,6 +850,14 @@ class EngineArgs:
load_group.add_argument(
"--safetensors-load-strategy", **load_kwargs["safetensors_load_strategy"]
)
load_group.add_argument(
"--safetensors-prefetch-num-threads",
**load_kwargs["safetensors_prefetch_num_threads"],
)
load_group.add_argument(
"--safetensors-prefetch-block-size",
**load_kwargs["safetensors_prefetch_block_size"],
)
load_group.add_argument(
"--model-loader-extra-config", **load_kwargs["model_loader_extra_config"]
)
@@ -1584,6 +1598,8 @@ class EngineArgs:
load_format=self.load_format,
download_dir=self.download_dir,
safetensors_load_strategy=self.safetensors_load_strategy,
safetensors_prefetch_num_threads=self.safetensors_prefetch_num_threads,
safetensors_prefetch_block_size=self.safetensors_prefetch_block_size,
model_loader_extra_config=self.model_loader_extra_config,
ignore_patterns=self.ignore_patterns,
use_tqdm_on_load=self.use_tqdm_on_load,
@@ -256,6 +256,12 @@ class DefaultModelLoader(BaseModelLoader):
self.load_config.use_tqdm_on_load,
self.load_config.safetensors_load_strategy,
local_expert_ids=self.local_expert_ids,
safetensors_prefetch_num_threads=(
self.load_config.safetensors_prefetch_num_threads
),
safetensors_prefetch_block_size=(
self.load_config.safetensors_prefetch_block_size
),
)
else:
if extra_config.get("enable_multithread_load"):
@@ -30,7 +30,11 @@ from transformers.utils import SAFE_WEIGHTS_INDEX_NAME
from vllm import envs
from vllm.config import ModelConfig
from vllm.config.load import LoadConfig
from vllm.config.load import (
DEFAULT_SAFETENSORS_PREFETCH_BLOCK_SIZE,
DEFAULT_SAFETENSORS_PREFETCH_NUM_THREADS,
LoadConfig,
)
from vllm.distributed import get_tensor_model_parallel_rank, get_world_group
from vllm.logger import init_logger
from vllm.model_executor.layers.quantization import (
@@ -810,40 +814,57 @@ def _get_fs_type(files: list[str]) -> str:
return ""
def _prefetch_checkpoint(file_path: str) -> None:
def _prefetch_checkpoint(
file_path: str,
block_size: int = DEFAULT_SAFETENSORS_PREFETCH_BLOCK_SIZE,
) -> None:
"""Prefetch a checkpoint file into the OS page cache.
Reads the file in 16MB blocks so the kernel caches its pages before
workers load the same file.
Reads the file in blocks so the kernel caches its pages before workers load
the same file.
"""
block_size = 16 * 1024 * 1024 # 16MB
if block_size < 1:
raise ValueError("safetensors prefetch block size must be >= 1")
with open(file_path, "rb") as f:
while f.read(block_size):
pass
def _prefetch_all_checkpoints(sorted_files: list[str]) -> None:
def _prefetch_all_checkpoints(
sorted_files: list[str],
num_prefetch_threads: int = DEFAULT_SAFETENSORS_PREFETCH_NUM_THREADS,
block_size: int = DEFAULT_SAFETENSORS_PREFETCH_BLOCK_SIZE,
) -> None:
"""Start prefetching checkpoint files into page cache in a background thread."""
if num_prefetch_threads < 1:
raise ValueError("safetensors prefetch num threads must be >= 1")
if block_size < 1:
raise ValueError("safetensors prefetch block size must be >= 1")
if torch.distributed.is_initialized():
rank = torch.distributed.get_rank()
world_size = torch.distributed.get_world_size()
else:
rank = 0
world_size = 1
num_prefetch_threads = 8
paths_to_prefetch = sorted_files[rank::world_size]
total_for_rank = len(paths_to_prefetch)
async def _prefetch_all() -> None:
semaphore = asyncio.Semaphore(num_prefetch_threads)
loop = asyncio.get_running_loop()
completed = 0
next_log_pct = 10
async def prefetch_one(path: str) -> None:
async def prefetch_one(
path: str,
executor: concurrent.futures.ThreadPoolExecutor,
) -> None:
nonlocal completed, next_log_pct
try:
async with semaphore:
await asyncio.to_thread(_prefetch_checkpoint, path)
await loop.run_in_executor(
executor, _prefetch_checkpoint, path, block_size
)
completed += 1
if total_for_rank > 0 and next_log_pct <= 100:
pct = 100 * completed / total_for_rank
@@ -860,7 +881,12 @@ def _prefetch_all_checkpoints(sorted_files: list[str]) -> None:
"Failed to prefetch checkpoint file %r.", path, exc_info=True
)
await asyncio.gather(*(prefetch_one(p) for p in paths_to_prefetch))
with concurrent.futures.ThreadPoolExecutor(
max_workers=num_prefetch_threads
) as executor:
await asyncio.gather(
*(prefetch_one(p, executor) for p in paths_to_prefetch)
)
def _run_prefetch() -> None:
start = time.perf_counter()
@@ -871,7 +897,12 @@ def _prefetch_all_checkpoints(sorted_files: list[str]) -> None:
elapsed,
)
logger.info("Prefetching checkpoint files into page cache started (in background)")
logger.info(
"Prefetching checkpoint files into page cache started "
"(in background, num_threads=%d, block_size=%d bytes)",
num_prefetch_threads,
block_size,
)
threading.Thread(target=_run_prefetch, daemon=True).start()
@@ -880,6 +911,9 @@ def safetensors_weights_iterator(
use_tqdm_on_load: bool,
safetensors_load_strategy: str | None = None,
local_expert_ids: set[int] | None = None,
*,
safetensors_prefetch_num_threads: int = DEFAULT_SAFETENSORS_PREFETCH_NUM_THREADS,
safetensors_prefetch_block_size: int = DEFAULT_SAFETENSORS_PREFETCH_BLOCK_SIZE,
) -> Generator[tuple[str, torch.Tensor], None, None]:
"""Iterate over the weights in the model safetensor files.
@@ -951,7 +985,11 @@ def safetensors_weights_iterator(
)
if should_prefetch:
_prefetch_all_checkpoints(sorted_files)
_prefetch_all_checkpoints(
sorted_files,
num_prefetch_threads=safetensors_prefetch_num_threads,
block_size=safetensors_prefetch_block_size,
)
leftover_state_dict: dict[str, torch.Tensor] = {}
for st_file in tqdm(