[Core] Use fastsafetensors ParallelLoader for weight loading (#40183)
Signed-off-by: Git Bisector <gitbisector@gmail.com> Signed-off-by: gitbisector <gitbisector@gmail.com> Signed-off-by: git bisector <gitbisector@gmail.com> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
co-authored by
Claude
Cyrus Leung
parent
f3858d5422
commit
9d808e2309
@@ -55,10 +55,9 @@ except ImportError:
|
||||
SafetensorsStreamer = runai_model_streamer.placeholder_attr("SafetensorsStreamer")
|
||||
|
||||
try:
|
||||
from fastsafetensors import SafeTensorsFileLoader, SingleGroup
|
||||
from fastsafetensors import SingleGroup
|
||||
except ImportError:
|
||||
fastsafetensors = PlaceholderModule("fastsafetensors")
|
||||
SafeTensorsFileLoader = fastsafetensors.placeholder_attr("SafeTensorsFileLoader")
|
||||
SingleGroup = fastsafetensors.placeholder_attr("SingleGroup")
|
||||
|
||||
from vllm.model_executor.layers.quantization.torchao import torchao_version_at_least
|
||||
@@ -1022,25 +1021,19 @@ def runai_safetensors_weights_iterator(
|
||||
yield name, tensor.clone()
|
||||
|
||||
|
||||
def _init_fastsafetensors_loader(
|
||||
pg: "torch.distributed.ProcessGroup",
|
||||
device: torch.device,
|
||||
f_list: list[str],
|
||||
*,
|
||||
nogds: bool = False,
|
||||
):
|
||||
loader = SafeTensorsFileLoader(pg, device, nogds=nogds)
|
||||
rank_file_map = {i: [f] for i, f in enumerate(f_list)}
|
||||
loader.add_filenames(rank_file_map)
|
||||
return loader
|
||||
|
||||
|
||||
def fastsafetensors_weights_iterator(
|
||||
hf_weights_files: list[str],
|
||||
use_tqdm_on_load: bool,
|
||||
) -> Generator[tuple[str, torch.Tensor], None, None]:
|
||||
"""Iterate over the weights in the model safetensor files
|
||||
using fastsafetensor library."""
|
||||
using fastsafetensor library.
|
||||
|
||||
Uses ParallelLoader for pipelined loading: the producer thread
|
||||
prepares metadata for the next shard while the consumer yields
|
||||
tensors from the current shard.
|
||||
"""
|
||||
from fastsafetensors.parallel_loader import ParallelLoader
|
||||
|
||||
if torch.distributed.is_initialized():
|
||||
pg = torch.distributed.group.WORLD
|
||||
else:
|
||||
@@ -1048,48 +1041,53 @@ def fastsafetensors_weights_iterator(
|
||||
|
||||
device = torch.device(f"cuda:{current_platform.current_device()}")
|
||||
hf_weights_files = sorted(hf_weights_files, key=_natural_sort_key)
|
||||
weight_files_sub_lists = [
|
||||
hf_weights_files[i : i + pg.size()]
|
||||
for i in range(0, len(hf_weights_files), pg.size())
|
||||
]
|
||||
|
||||
# Use nogds=True for TP > 1 to avoid cuFileDriverOpen() which
|
||||
# initializes the GDS DMA subsystem for all visible GPUs, creating
|
||||
# unwanted CUDA contexts on every device.
|
||||
nogds = pg.size() > 1
|
||||
|
||||
for f_list in tqdm(
|
||||
weight_files_sub_lists,
|
||||
desc="Loading safetensors using Fastsafetensor loader",
|
||||
disable=not enable_tqdm(use_tqdm_on_load),
|
||||
bar_format=_BAR_FORMAT,
|
||||
):
|
||||
loader = _init_fastsafetensors_loader(pg, device, f_list, nogds=nogds)
|
||||
queue_size = envs.VLLM_FASTSAFETENSORS_QUEUE_SIZE
|
||||
tqdm_enabled = enable_tqdm(use_tqdm_on_load)
|
||||
|
||||
def _make_loader(nogds: bool) -> "ParallelLoader":
|
||||
return ParallelLoader(
|
||||
pg=pg,
|
||||
hf_weights_files=hf_weights_files,
|
||||
queue_size=queue_size,
|
||||
use_tqdm_on_load=tqdm_enabled,
|
||||
device=str(device),
|
||||
nogds=nogds,
|
||||
)
|
||||
|
||||
# GDS can fail either at construction or lazily inside the producer
|
||||
# thread during iteration (e.g. cuFileHandleRegister returning
|
||||
# CU_FILE_HANDLE_NOT_REGISTERED on a filesystem without GDS support).
|
||||
# Catch both and fall back to nogds, but only before yielding any
|
||||
# tensor -- restarting mid-stream would reload earlier shards.
|
||||
pl = None
|
||||
yielded = False
|
||||
try:
|
||||
try:
|
||||
try:
|
||||
fb = loader.copy_files_to_device()
|
||||
except RuntimeError as e:
|
||||
if "gds" not in str(e):
|
||||
raise
|
||||
|
||||
loader.close()
|
||||
nogds = True
|
||||
logger.warning_once(
|
||||
"GDS not enabled, setting `nogds=True`.\n"
|
||||
"For more information, see: https://github.com/foundation-model-stack/fastsafetensors?tab=readme-ov-file#basic-api-usages"
|
||||
)
|
||||
loader = _init_fastsafetensors_loader(pg, device, f_list, nogds=nogds)
|
||||
fb = loader.copy_files_to_device()
|
||||
|
||||
try:
|
||||
keys = list(fb.key_to_rank_lidx.keys())
|
||||
for k in keys:
|
||||
t = fb.get_tensor(k)
|
||||
yield k, t
|
||||
finally:
|
||||
fb.close()
|
||||
finally:
|
||||
loader.close()
|
||||
pl = _make_loader(nogds)
|
||||
for name, tensor in pl.iterate_weights():
|
||||
yielded = True
|
||||
yield name, tensor
|
||||
except RuntimeError as e:
|
||||
if nogds or yielded or "gds" not in str(e):
|
||||
raise
|
||||
logger.warning_once(
|
||||
"GDS not enabled, setting `nogds=True`.\n"
|
||||
"For more information, see: https://github.com/foundation-model-stack/"
|
||||
"fastsafetensors?tab=readme-ov-file#basic-api-usages"
|
||||
)
|
||||
if pl is not None:
|
||||
pl.close()
|
||||
pl = _make_loader(nogds=True)
|
||||
yield from pl.iterate_weights()
|
||||
finally:
|
||||
if pl is not None:
|
||||
pl.close()
|
||||
|
||||
|
||||
def instanttensor_weights_iterator(
|
||||
|
||||
Reference in New Issue
Block a user