[Core] Use fastsafetensors ParallelLoader for weight loading (#40183)

Signed-off-by: Git Bisector <gitbisector@gmail.com>
Signed-off-by: gitbisector <gitbisector@gmail.com>
Signed-off-by: git bisector <gitbisector@gmail.com>
Co-authored-by: Claude <noreply@anthropic.com>
Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
gitbisector
2026-06-15 22:32:05 -07:00
committed by GitHub
co-authored by Claude Cyrus Leung
parent f3858d5422
commit 9d808e2309
3 changed files with 68 additions and 56 deletions
@@ -55,10 +55,9 @@ except ImportError:
SafetensorsStreamer = runai_model_streamer.placeholder_attr("SafetensorsStreamer")
try:
from fastsafetensors import SafeTensorsFileLoader, SingleGroup
from fastsafetensors import SingleGroup
except ImportError:
fastsafetensors = PlaceholderModule("fastsafetensors")
SafeTensorsFileLoader = fastsafetensors.placeholder_attr("SafeTensorsFileLoader")
SingleGroup = fastsafetensors.placeholder_attr("SingleGroup")
from vllm.model_executor.layers.quantization.torchao import torchao_version_at_least
@@ -1022,25 +1021,19 @@ def runai_safetensors_weights_iterator(
yield name, tensor.clone()
def _init_fastsafetensors_loader(
pg: "torch.distributed.ProcessGroup",
device: torch.device,
f_list: list[str],
*,
nogds: bool = False,
):
loader = SafeTensorsFileLoader(pg, device, nogds=nogds)
rank_file_map = {i: [f] for i, f in enumerate(f_list)}
loader.add_filenames(rank_file_map)
return loader
def fastsafetensors_weights_iterator(
hf_weights_files: list[str],
use_tqdm_on_load: bool,
) -> Generator[tuple[str, torch.Tensor], None, None]:
"""Iterate over the weights in the model safetensor files
using fastsafetensor library."""
using fastsafetensor library.
Uses ParallelLoader for pipelined loading: the producer thread
prepares metadata for the next shard while the consumer yields
tensors from the current shard.
"""
from fastsafetensors.parallel_loader import ParallelLoader
if torch.distributed.is_initialized():
pg = torch.distributed.group.WORLD
else:
@@ -1048,48 +1041,53 @@ def fastsafetensors_weights_iterator(
device = torch.device(f"cuda:{current_platform.current_device()}")
hf_weights_files = sorted(hf_weights_files, key=_natural_sort_key)
weight_files_sub_lists = [
hf_weights_files[i : i + pg.size()]
for i in range(0, len(hf_weights_files), pg.size())
]
# Use nogds=True for TP > 1 to avoid cuFileDriverOpen() which
# initializes the GDS DMA subsystem for all visible GPUs, creating
# unwanted CUDA contexts on every device.
nogds = pg.size() > 1
for f_list in tqdm(
weight_files_sub_lists,
desc="Loading safetensors using Fastsafetensor loader",
disable=not enable_tqdm(use_tqdm_on_load),
bar_format=_BAR_FORMAT,
):
loader = _init_fastsafetensors_loader(pg, device, f_list, nogds=nogds)
queue_size = envs.VLLM_FASTSAFETENSORS_QUEUE_SIZE
tqdm_enabled = enable_tqdm(use_tqdm_on_load)
def _make_loader(nogds: bool) -> "ParallelLoader":
return ParallelLoader(
pg=pg,
hf_weights_files=hf_weights_files,
queue_size=queue_size,
use_tqdm_on_load=tqdm_enabled,
device=str(device),
nogds=nogds,
)
# GDS can fail either at construction or lazily inside the producer
# thread during iteration (e.g. cuFileHandleRegister returning
# CU_FILE_HANDLE_NOT_REGISTERED on a filesystem without GDS support).
# Catch both and fall back to nogds, but only before yielding any
# tensor -- restarting mid-stream would reload earlier shards.
pl = None
yielded = False
try:
try:
try:
fb = loader.copy_files_to_device()
except RuntimeError as e:
if "gds" not in str(e):
raise
loader.close()
nogds = True
logger.warning_once(
"GDS not enabled, setting `nogds=True`.\n"
"For more information, see: https://github.com/foundation-model-stack/fastsafetensors?tab=readme-ov-file#basic-api-usages"
)
loader = _init_fastsafetensors_loader(pg, device, f_list, nogds=nogds)
fb = loader.copy_files_to_device()
try:
keys = list(fb.key_to_rank_lidx.keys())
for k in keys:
t = fb.get_tensor(k)
yield k, t
finally:
fb.close()
finally:
loader.close()
pl = _make_loader(nogds)
for name, tensor in pl.iterate_weights():
yielded = True
yield name, tensor
except RuntimeError as e:
if nogds or yielded or "gds" not in str(e):
raise
logger.warning_once(
"GDS not enabled, setting `nogds=True`.\n"
"For more information, see: https://github.com/foundation-model-stack/"
"fastsafetensors?tab=readme-ov-file#basic-api-usages"
)
if pl is not None:
pl.close()
pl = _make_loader(nogds=True)
yield from pl.iterate_weights()
finally:
if pl is not None:
pl.close()
def instanttensor_weights_iterator(