[Feat] Adds runai distributed streamer (#27230)
Signed-off-by: bbartels <benjamin@bartels.dev> Signed-off-by: Benjamin Bartels <benjamin@bartels.dev> Co-authored-by: omer-dayan <omdayan@nvidia.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
co-authored by
omer-dayan
Cyrus Leung
parent
2ce5c5d3d6
commit
17d055f527
@@ -27,9 +27,16 @@ class RunaiModelStreamerLoader(BaseModelLoader):
|
||||
|
||||
def __init__(self, load_config: LoadConfig):
|
||||
super().__init__(load_config)
|
||||
|
||||
self._is_distributed = False
|
||||
if load_config.model_loader_extra_config:
|
||||
extra_config = load_config.model_loader_extra_config
|
||||
|
||||
if "distributed" in extra_config and isinstance(
|
||||
extra_config.get("distributed"), bool
|
||||
):
|
||||
self._is_distributed = extra_config.get("distributed")
|
||||
|
||||
if "concurrency" in extra_config and isinstance(
|
||||
extra_config.get("concurrency"), int
|
||||
):
|
||||
@@ -92,8 +99,7 @@ class RunaiModelStreamerLoader(BaseModelLoader):
|
||||
"""Get an iterator for the model weights based on the load format."""
|
||||
hf_weights_files = self._prepare_weights(model_or_path, revision)
|
||||
return runai_safetensors_weights_iterator(
|
||||
hf_weights_files,
|
||||
self.load_config.use_tqdm_on_load,
|
||||
hf_weights_files, self.load_config.use_tqdm_on_load, self._is_distributed
|
||||
)
|
||||
|
||||
def download_model(self, model_config: ModelConfig) -> None:
|
||||
|
||||
@@ -657,10 +657,22 @@ def multi_thread_safetensors_weights_iterator(
|
||||
def runai_safetensors_weights_iterator(
|
||||
hf_weights_files: list[str],
|
||||
use_tqdm_on_load: bool,
|
||||
is_distributed: bool = False,
|
||||
) -> Generator[tuple[str, torch.Tensor], None, None]:
|
||||
"""Iterate over the weights in the model safetensor files."""
|
||||
with SafetensorsStreamer() as streamer:
|
||||
streamer.stream_files(hf_weights_files)
|
||||
is_cuda_alike = current_platform.is_cuda_alike()
|
||||
device = (
|
||||
f"cuda:{current_platform.current_device()}"
|
||||
if is_distributed and is_cuda_alike
|
||||
else "cpu"
|
||||
)
|
||||
|
||||
streamer.stream_files(
|
||||
hf_weights_files,
|
||||
device=device,
|
||||
is_distributed=is_distributed,
|
||||
)
|
||||
total_tensors = sum(
|
||||
len(tensors_meta)
|
||||
for tensors_meta in streamer.files_to_tensors_metadata.values()
|
||||
@@ -672,6 +684,7 @@ def runai_safetensors_weights_iterator(
|
||||
desc="Loading safetensors using Runai Model Streamer",
|
||||
bar_format=_BAR_FORMAT,
|
||||
disable=not enable_tqdm(use_tqdm_on_load),
|
||||
mininterval=2,
|
||||
)
|
||||
|
||||
yield from tensor_iter
|
||||
|
||||
Reference in New Issue
Block a user