[Performance] Make safetensors checkpoint prefetch settings configurable (#41499)

Signed-off-by: Mohammad Miadh Angkad <MAngkad.BSDSBA2027@aim.edu>
This commit is contained in:
Mohammad Miadh Angkad
2026-05-10 15:55:15 +00:00
committed by GitHub
parent f396bee56f
commit 21943d4c25
4 changed files with 87 additions and 15 deletions
@@ -256,6 +256,12 @@ class DefaultModelLoader(BaseModelLoader):
self.load_config.use_tqdm_on_load,
self.load_config.safetensors_load_strategy,
local_expert_ids=self.local_expert_ids,
safetensors_prefetch_num_threads=(
self.load_config.safetensors_prefetch_num_threads
),
safetensors_prefetch_block_size=(
self.load_config.safetensors_prefetch_block_size
),
)
else:
if extra_config.get("enable_multithread_load"):
@@ -30,7 +30,11 @@ from transformers.utils import SAFE_WEIGHTS_INDEX_NAME
from vllm import envs
from vllm.config import ModelConfig
from vllm.config.load import LoadConfig
from vllm.config.load import (
DEFAULT_SAFETENSORS_PREFETCH_BLOCK_SIZE,
DEFAULT_SAFETENSORS_PREFETCH_NUM_THREADS,
LoadConfig,
)
from vllm.distributed import get_tensor_model_parallel_rank, get_world_group
from vllm.logger import init_logger
from vllm.model_executor.layers.quantization import (
@@ -810,40 +814,57 @@ def _get_fs_type(files: list[str]) -> str:
return ""
def _prefetch_checkpoint(file_path: str) -> None:
def _prefetch_checkpoint(
file_path: str,
block_size: int = DEFAULT_SAFETENSORS_PREFETCH_BLOCK_SIZE,
) -> None:
"""Prefetch a checkpoint file into the OS page cache.
Reads the file in 16MB blocks so the kernel caches its pages before
workers load the same file.
Reads the file in blocks so the kernel caches its pages before workers load
the same file.
"""
block_size = 16 * 1024 * 1024 # 16MB
if block_size < 1:
raise ValueError("safetensors prefetch block size must be >= 1")
with open(file_path, "rb") as f:
while f.read(block_size):
pass
def _prefetch_all_checkpoints(sorted_files: list[str]) -> None:
def _prefetch_all_checkpoints(
sorted_files: list[str],
num_prefetch_threads: int = DEFAULT_SAFETENSORS_PREFETCH_NUM_THREADS,
block_size: int = DEFAULT_SAFETENSORS_PREFETCH_BLOCK_SIZE,
) -> None:
"""Start prefetching checkpoint files into page cache in a background thread."""
if num_prefetch_threads < 1:
raise ValueError("safetensors prefetch num threads must be >= 1")
if block_size < 1:
raise ValueError("safetensors prefetch block size must be >= 1")
if torch.distributed.is_initialized():
rank = torch.distributed.get_rank()
world_size = torch.distributed.get_world_size()
else:
rank = 0
world_size = 1
num_prefetch_threads = 8
paths_to_prefetch = sorted_files[rank::world_size]
total_for_rank = len(paths_to_prefetch)
async def _prefetch_all() -> None:
semaphore = asyncio.Semaphore(num_prefetch_threads)
loop = asyncio.get_running_loop()
completed = 0
next_log_pct = 10
async def prefetch_one(path: str) -> None:
async def prefetch_one(
path: str,
executor: concurrent.futures.ThreadPoolExecutor,
) -> None:
nonlocal completed, next_log_pct
try:
async with semaphore:
await asyncio.to_thread(_prefetch_checkpoint, path)
await loop.run_in_executor(
executor, _prefetch_checkpoint, path, block_size
)
completed += 1
if total_for_rank > 0 and next_log_pct <= 100:
pct = 100 * completed / total_for_rank
@@ -860,7 +881,12 @@ def _prefetch_all_checkpoints(sorted_files: list[str]) -> None:
"Failed to prefetch checkpoint file %r.", path, exc_info=True
)
await asyncio.gather(*(prefetch_one(p) for p in paths_to_prefetch))
with concurrent.futures.ThreadPoolExecutor(
max_workers=num_prefetch_threads
) as executor:
await asyncio.gather(
*(prefetch_one(p, executor) for p in paths_to_prefetch)
)
def _run_prefetch() -> None:
start = time.perf_counter()
@@ -871,7 +897,12 @@ def _prefetch_all_checkpoints(sorted_files: list[str]) -> None:
elapsed,
)
logger.info("Prefetching checkpoint files into page cache started (in background)")
logger.info(
"Prefetching checkpoint files into page cache started "
"(in background, num_threads=%d, block_size=%d bytes)",
num_prefetch_threads,
block_size,
)
threading.Thread(target=_run_prefetch, daemon=True).start()
@@ -880,6 +911,9 @@ def safetensors_weights_iterator(
use_tqdm_on_load: bool,
safetensors_load_strategy: str | None = None,
local_expert_ids: set[int] | None = None,
*,
safetensors_prefetch_num_threads: int = DEFAULT_SAFETENSORS_PREFETCH_NUM_THREADS,
safetensors_prefetch_block_size: int = DEFAULT_SAFETENSORS_PREFETCH_BLOCK_SIZE,
) -> Generator[tuple[str, torch.Tensor], None, None]:
"""Iterate over the weights in the model safetensor files.
@@ -951,7 +985,11 @@ def safetensors_weights_iterator(
)
if should_prefetch:
_prefetch_all_checkpoints(sorted_files)
_prefetch_all_checkpoints(
sorted_files,
num_prefetch_threads=safetensors_prefetch_num_threads,
block_size=safetensors_prefetch_block_size,
)
leftover_state_dict: dict[str, torch.Tensor] = {}
for st_file in tqdm(