[Performance] Make safetensors checkpoint prefetch settings configurable (#41499)
Signed-off-by: Mohammad Miadh Angkad <MAngkad.BSDSBA2027@aim.edu>
This commit is contained in:
@@ -30,7 +30,11 @@ from transformers.utils import SAFE_WEIGHTS_INDEX_NAME
|
||||
|
||||
from vllm import envs
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.config.load import LoadConfig
|
||||
from vllm.config.load import (
|
||||
DEFAULT_SAFETENSORS_PREFETCH_BLOCK_SIZE,
|
||||
DEFAULT_SAFETENSORS_PREFETCH_NUM_THREADS,
|
||||
LoadConfig,
|
||||
)
|
||||
from vllm.distributed import get_tensor_model_parallel_rank, get_world_group
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.quantization import (
|
||||
@@ -810,40 +814,57 @@ def _get_fs_type(files: list[str]) -> str:
|
||||
return ""
|
||||
|
||||
|
||||
def _prefetch_checkpoint(file_path: str) -> None:
|
||||
def _prefetch_checkpoint(
|
||||
file_path: str,
|
||||
block_size: int = DEFAULT_SAFETENSORS_PREFETCH_BLOCK_SIZE,
|
||||
) -> None:
|
||||
"""Prefetch a checkpoint file into the OS page cache.
|
||||
|
||||
Reads the file in 16MB blocks so the kernel caches its pages before
|
||||
workers load the same file.
|
||||
Reads the file in blocks so the kernel caches its pages before workers load
|
||||
the same file.
|
||||
"""
|
||||
block_size = 16 * 1024 * 1024 # 16MB
|
||||
if block_size < 1:
|
||||
raise ValueError("safetensors prefetch block size must be >= 1")
|
||||
|
||||
with open(file_path, "rb") as f:
|
||||
while f.read(block_size):
|
||||
pass
|
||||
|
||||
|
||||
def _prefetch_all_checkpoints(sorted_files: list[str]) -> None:
|
||||
def _prefetch_all_checkpoints(
|
||||
sorted_files: list[str],
|
||||
num_prefetch_threads: int = DEFAULT_SAFETENSORS_PREFETCH_NUM_THREADS,
|
||||
block_size: int = DEFAULT_SAFETENSORS_PREFETCH_BLOCK_SIZE,
|
||||
) -> None:
|
||||
"""Start prefetching checkpoint files into page cache in a background thread."""
|
||||
if num_prefetch_threads < 1:
|
||||
raise ValueError("safetensors prefetch num threads must be >= 1")
|
||||
if block_size < 1:
|
||||
raise ValueError("safetensors prefetch block size must be >= 1")
|
||||
|
||||
if torch.distributed.is_initialized():
|
||||
rank = torch.distributed.get_rank()
|
||||
world_size = torch.distributed.get_world_size()
|
||||
else:
|
||||
rank = 0
|
||||
world_size = 1
|
||||
num_prefetch_threads = 8
|
||||
paths_to_prefetch = sorted_files[rank::world_size]
|
||||
total_for_rank = len(paths_to_prefetch)
|
||||
|
||||
async def _prefetch_all() -> None:
|
||||
semaphore = asyncio.Semaphore(num_prefetch_threads)
|
||||
loop = asyncio.get_running_loop()
|
||||
completed = 0
|
||||
next_log_pct = 10
|
||||
|
||||
async def prefetch_one(path: str) -> None:
|
||||
async def prefetch_one(
|
||||
path: str,
|
||||
executor: concurrent.futures.ThreadPoolExecutor,
|
||||
) -> None:
|
||||
nonlocal completed, next_log_pct
|
||||
try:
|
||||
async with semaphore:
|
||||
await asyncio.to_thread(_prefetch_checkpoint, path)
|
||||
await loop.run_in_executor(
|
||||
executor, _prefetch_checkpoint, path, block_size
|
||||
)
|
||||
completed += 1
|
||||
if total_for_rank > 0 and next_log_pct <= 100:
|
||||
pct = 100 * completed / total_for_rank
|
||||
@@ -860,7 +881,12 @@ def _prefetch_all_checkpoints(sorted_files: list[str]) -> None:
|
||||
"Failed to prefetch checkpoint file %r.", path, exc_info=True
|
||||
)
|
||||
|
||||
await asyncio.gather(*(prefetch_one(p) for p in paths_to_prefetch))
|
||||
with concurrent.futures.ThreadPoolExecutor(
|
||||
max_workers=num_prefetch_threads
|
||||
) as executor:
|
||||
await asyncio.gather(
|
||||
*(prefetch_one(p, executor) for p in paths_to_prefetch)
|
||||
)
|
||||
|
||||
def _run_prefetch() -> None:
|
||||
start = time.perf_counter()
|
||||
@@ -871,7 +897,12 @@ def _prefetch_all_checkpoints(sorted_files: list[str]) -> None:
|
||||
elapsed,
|
||||
)
|
||||
|
||||
logger.info("Prefetching checkpoint files into page cache started (in background)")
|
||||
logger.info(
|
||||
"Prefetching checkpoint files into page cache started "
|
||||
"(in background, num_threads=%d, block_size=%d bytes)",
|
||||
num_prefetch_threads,
|
||||
block_size,
|
||||
)
|
||||
threading.Thread(target=_run_prefetch, daemon=True).start()
|
||||
|
||||
|
||||
@@ -880,6 +911,9 @@ def safetensors_weights_iterator(
|
||||
use_tqdm_on_load: bool,
|
||||
safetensors_load_strategy: str | None = None,
|
||||
local_expert_ids: set[int] | None = None,
|
||||
*,
|
||||
safetensors_prefetch_num_threads: int = DEFAULT_SAFETENSORS_PREFETCH_NUM_THREADS,
|
||||
safetensors_prefetch_block_size: int = DEFAULT_SAFETENSORS_PREFETCH_BLOCK_SIZE,
|
||||
) -> Generator[tuple[str, torch.Tensor], None, None]:
|
||||
"""Iterate over the weights in the model safetensor files.
|
||||
|
||||
@@ -951,7 +985,11 @@ def safetensors_weights_iterator(
|
||||
)
|
||||
|
||||
if should_prefetch:
|
||||
_prefetch_all_checkpoints(sorted_files)
|
||||
_prefetch_all_checkpoints(
|
||||
sorted_files,
|
||||
num_prefetch_threads=safetensors_prefetch_num_threads,
|
||||
block_size=safetensors_prefetch_block_size,
|
||||
)
|
||||
|
||||
leftover_state_dict: dict[str, torch.Tensor] = {}
|
||||
for st_file in tqdm(
|
||||
|
||||
Reference in New Issue
Block a user