[Performance] Make safetensors checkpoint prefetch settings configurable (#41499)
Signed-off-by: Mohammad Miadh Angkad <MAngkad.BSDSBA2027@aim.edu>
This commit is contained in:
@@ -9,6 +9,9 @@ from vllm.config.utils import config
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.hashing import safe_hash
|
||||
|
||||
DEFAULT_SAFETENSORS_PREFETCH_NUM_THREADS = 8
|
||||
DEFAULT_SAFETENSORS_PREFETCH_BLOCK_SIZE = 16 * 1024 * 1024
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.model_executor.model_loader import LoadFormats
|
||||
from vllm.model_executor.model_loader.tensorizer import TensorizerConfig
|
||||
@@ -79,6 +82,15 @@ class LoadConfig:
|
||||
was quantized using torchao and saved using safetensors.
|
||||
Needs `torchao >= 0.14.0`.
|
||||
"""
|
||||
safetensors_prefetch_num_threads: int = Field(
|
||||
default=DEFAULT_SAFETENSORS_PREFETCH_NUM_THREADS, ge=1
|
||||
)
|
||||
"""Number of worker threads used to prefetch safetensors checkpoint files
|
||||
into the OS page cache when safetensors prefetching is enabled."""
|
||||
safetensors_prefetch_block_size: int = Field(
|
||||
default=DEFAULT_SAFETENSORS_PREFETCH_BLOCK_SIZE, ge=1
|
||||
)
|
||||
"""Read size in bytes for each safetensors checkpoint file prefetch."""
|
||||
model_loader_extra_config: dict | TensorizerConfig = Field(default_factory=dict)
|
||||
"""Extra config for model loader. This will be passed to the model loader
|
||||
corresponding to the chosen load_format."""
|
||||
|
||||
@@ -355,7 +355,11 @@ def _compute_kwargs(cls: ConfigType) -> dict[str, dict[str, Any]]:
|
||||
if name == "max_model_len":
|
||||
kwargs[name]["type"] = human_readable_int_or_auto
|
||||
kwargs[name]["help"] += f"\n\n{human_readable_int_or_auto.__doc__}"
|
||||
elif name in ("max_num_batched_tokens", "kv_cache_memory_bytes"):
|
||||
elif name in (
|
||||
"max_num_batched_tokens",
|
||||
"kv_cache_memory_bytes",
|
||||
"safetensors_prefetch_block_size",
|
||||
):
|
||||
kwargs[name]["type"] = human_readable_int
|
||||
kwargs[name]["help"] += f"\n\n{human_readable_int.__doc__}"
|
||||
else:
|
||||
@@ -424,6 +428,8 @@ class EngineArgs:
|
||||
allowed_media_domains: list[str] | None = ModelConfig.allowed_media_domains
|
||||
download_dir: str | None = LoadConfig.download_dir
|
||||
safetensors_load_strategy: str | None = LoadConfig.safetensors_load_strategy
|
||||
safetensors_prefetch_num_threads: int = LoadConfig.safetensors_prefetch_num_threads
|
||||
safetensors_prefetch_block_size: int = LoadConfig.safetensors_prefetch_block_size
|
||||
load_format: str | LoadFormats = LoadConfig.load_format
|
||||
config_format: str = ModelConfig.config_format
|
||||
dtype: ModelDType = ModelConfig.dtype
|
||||
@@ -844,6 +850,14 @@ class EngineArgs:
|
||||
load_group.add_argument(
|
||||
"--safetensors-load-strategy", **load_kwargs["safetensors_load_strategy"]
|
||||
)
|
||||
load_group.add_argument(
|
||||
"--safetensors-prefetch-num-threads",
|
||||
**load_kwargs["safetensors_prefetch_num_threads"],
|
||||
)
|
||||
load_group.add_argument(
|
||||
"--safetensors-prefetch-block-size",
|
||||
**load_kwargs["safetensors_prefetch_block_size"],
|
||||
)
|
||||
load_group.add_argument(
|
||||
"--model-loader-extra-config", **load_kwargs["model_loader_extra_config"]
|
||||
)
|
||||
@@ -1584,6 +1598,8 @@ class EngineArgs:
|
||||
load_format=self.load_format,
|
||||
download_dir=self.download_dir,
|
||||
safetensors_load_strategy=self.safetensors_load_strategy,
|
||||
safetensors_prefetch_num_threads=self.safetensors_prefetch_num_threads,
|
||||
safetensors_prefetch_block_size=self.safetensors_prefetch_block_size,
|
||||
model_loader_extra_config=self.model_loader_extra_config,
|
||||
ignore_patterns=self.ignore_patterns,
|
||||
use_tqdm_on_load=self.use_tqdm_on_load,
|
||||
|
||||
@@ -256,6 +256,12 @@ class DefaultModelLoader(BaseModelLoader):
|
||||
self.load_config.use_tqdm_on_load,
|
||||
self.load_config.safetensors_load_strategy,
|
||||
local_expert_ids=self.local_expert_ids,
|
||||
safetensors_prefetch_num_threads=(
|
||||
self.load_config.safetensors_prefetch_num_threads
|
||||
),
|
||||
safetensors_prefetch_block_size=(
|
||||
self.load_config.safetensors_prefetch_block_size
|
||||
),
|
||||
)
|
||||
else:
|
||||
if extra_config.get("enable_multithread_load"):
|
||||
|
||||
@@ -30,7 +30,11 @@ from transformers.utils import SAFE_WEIGHTS_INDEX_NAME
|
||||
|
||||
from vllm import envs
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.config.load import LoadConfig
|
||||
from vllm.config.load import (
|
||||
DEFAULT_SAFETENSORS_PREFETCH_BLOCK_SIZE,
|
||||
DEFAULT_SAFETENSORS_PREFETCH_NUM_THREADS,
|
||||
LoadConfig,
|
||||
)
|
||||
from vllm.distributed import get_tensor_model_parallel_rank, get_world_group
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.quantization import (
|
||||
@@ -810,40 +814,57 @@ def _get_fs_type(files: list[str]) -> str:
|
||||
return ""
|
||||
|
||||
|
||||
def _prefetch_checkpoint(file_path: str) -> None:
|
||||
def _prefetch_checkpoint(
|
||||
file_path: str,
|
||||
block_size: int = DEFAULT_SAFETENSORS_PREFETCH_BLOCK_SIZE,
|
||||
) -> None:
|
||||
"""Prefetch a checkpoint file into the OS page cache.
|
||||
|
||||
Reads the file in 16MB blocks so the kernel caches its pages before
|
||||
workers load the same file.
|
||||
Reads the file in blocks so the kernel caches its pages before workers load
|
||||
the same file.
|
||||
"""
|
||||
block_size = 16 * 1024 * 1024 # 16MB
|
||||
if block_size < 1:
|
||||
raise ValueError("safetensors prefetch block size must be >= 1")
|
||||
|
||||
with open(file_path, "rb") as f:
|
||||
while f.read(block_size):
|
||||
pass
|
||||
|
||||
|
||||
def _prefetch_all_checkpoints(sorted_files: list[str]) -> None:
|
||||
def _prefetch_all_checkpoints(
|
||||
sorted_files: list[str],
|
||||
num_prefetch_threads: int = DEFAULT_SAFETENSORS_PREFETCH_NUM_THREADS,
|
||||
block_size: int = DEFAULT_SAFETENSORS_PREFETCH_BLOCK_SIZE,
|
||||
) -> None:
|
||||
"""Start prefetching checkpoint files into page cache in a background thread."""
|
||||
if num_prefetch_threads < 1:
|
||||
raise ValueError("safetensors prefetch num threads must be >= 1")
|
||||
if block_size < 1:
|
||||
raise ValueError("safetensors prefetch block size must be >= 1")
|
||||
|
||||
if torch.distributed.is_initialized():
|
||||
rank = torch.distributed.get_rank()
|
||||
world_size = torch.distributed.get_world_size()
|
||||
else:
|
||||
rank = 0
|
||||
world_size = 1
|
||||
num_prefetch_threads = 8
|
||||
paths_to_prefetch = sorted_files[rank::world_size]
|
||||
total_for_rank = len(paths_to_prefetch)
|
||||
|
||||
async def _prefetch_all() -> None:
|
||||
semaphore = asyncio.Semaphore(num_prefetch_threads)
|
||||
loop = asyncio.get_running_loop()
|
||||
completed = 0
|
||||
next_log_pct = 10
|
||||
|
||||
async def prefetch_one(path: str) -> None:
|
||||
async def prefetch_one(
|
||||
path: str,
|
||||
executor: concurrent.futures.ThreadPoolExecutor,
|
||||
) -> None:
|
||||
nonlocal completed, next_log_pct
|
||||
try:
|
||||
async with semaphore:
|
||||
await asyncio.to_thread(_prefetch_checkpoint, path)
|
||||
await loop.run_in_executor(
|
||||
executor, _prefetch_checkpoint, path, block_size
|
||||
)
|
||||
completed += 1
|
||||
if total_for_rank > 0 and next_log_pct <= 100:
|
||||
pct = 100 * completed / total_for_rank
|
||||
@@ -860,7 +881,12 @@ def _prefetch_all_checkpoints(sorted_files: list[str]) -> None:
|
||||
"Failed to prefetch checkpoint file %r.", path, exc_info=True
|
||||
)
|
||||
|
||||
await asyncio.gather(*(prefetch_one(p) for p in paths_to_prefetch))
|
||||
with concurrent.futures.ThreadPoolExecutor(
|
||||
max_workers=num_prefetch_threads
|
||||
) as executor:
|
||||
await asyncio.gather(
|
||||
*(prefetch_one(p, executor) for p in paths_to_prefetch)
|
||||
)
|
||||
|
||||
def _run_prefetch() -> None:
|
||||
start = time.perf_counter()
|
||||
@@ -871,7 +897,12 @@ def _prefetch_all_checkpoints(sorted_files: list[str]) -> None:
|
||||
elapsed,
|
||||
)
|
||||
|
||||
logger.info("Prefetching checkpoint files into page cache started (in background)")
|
||||
logger.info(
|
||||
"Prefetching checkpoint files into page cache started "
|
||||
"(in background, num_threads=%d, block_size=%d bytes)",
|
||||
num_prefetch_threads,
|
||||
block_size,
|
||||
)
|
||||
threading.Thread(target=_run_prefetch, daemon=True).start()
|
||||
|
||||
|
||||
@@ -880,6 +911,9 @@ def safetensors_weights_iterator(
|
||||
use_tqdm_on_load: bool,
|
||||
safetensors_load_strategy: str | None = None,
|
||||
local_expert_ids: set[int] | None = None,
|
||||
*,
|
||||
safetensors_prefetch_num_threads: int = DEFAULT_SAFETENSORS_PREFETCH_NUM_THREADS,
|
||||
safetensors_prefetch_block_size: int = DEFAULT_SAFETENSORS_PREFETCH_BLOCK_SIZE,
|
||||
) -> Generator[tuple[str, torch.Tensor], None, None]:
|
||||
"""Iterate over the weights in the model safetensor files.
|
||||
|
||||
@@ -951,7 +985,11 @@ def safetensors_weights_iterator(
|
||||
)
|
||||
|
||||
if should_prefetch:
|
||||
_prefetch_all_checkpoints(sorted_files)
|
||||
_prefetch_all_checkpoints(
|
||||
sorted_files,
|
||||
num_prefetch_threads=safetensors_prefetch_num_threads,
|
||||
block_size=safetensors_prefetch_block_size,
|
||||
)
|
||||
|
||||
leftover_state_dict: dict[str, torch.Tensor] = {}
|
||||
for st_file in tqdm(
|
||||
|
||||
Reference in New Issue
Block a user