[Loader] Improve InstantTensor loading (#46868)
Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
c4cd2bd544
commit
cc25f028b7
@@ -1120,7 +1120,7 @@ def instanttensor_weights_iterator(
|
||||
import instanttensor
|
||||
except ImportError as e:
|
||||
raise ImportError(
|
||||
"Please install instanttensor via `pip install instanttensor`"
|
||||
"Please install instanttensor via `pip install vllm[instanttensor]`"
|
||||
) from e
|
||||
|
||||
if not current_platform.is_cuda():
|
||||
@@ -1136,18 +1136,33 @@ def instanttensor_weights_iterator(
|
||||
|
||||
device = current_platform.current_device()
|
||||
|
||||
# copy=True yields tensors that own their memory, staying valid after the
|
||||
# context exits or InstantTensor reuses its buffer.
|
||||
with instanttensor.safe_open(
|
||||
hf_weights_files, framework="pt", device=device, process_group=process_group
|
||||
hf_weights_files,
|
||||
framework="pt",
|
||||
device=device,
|
||||
process_group=process_group,
|
||||
copy=True,
|
||||
) as f:
|
||||
yield from tqdm(
|
||||
f.tensors(),
|
||||
# Track bytes so the bar reports load throughput (GB/s).
|
||||
pbar = tqdm(
|
||||
total=f.total_tensor_size,
|
||||
desc="Loading safetensors using InstantTensor loader",
|
||||
disable=not enable_tqdm(use_tqdm_on_load),
|
||||
bar_format=_BAR_FORMAT,
|
||||
position=tqdm._get_free_pos(),
|
||||
total=len(f.keys()),
|
||||
unit="B",
|
||||
unit_scale=True,
|
||||
unit_divisor=1024,
|
||||
mininterval=1.0,
|
||||
)
|
||||
try:
|
||||
for name, tensor in f.tensors():
|
||||
pbar.update(tensor.numel() * tensor.element_size())
|
||||
yield name, tensor
|
||||
finally:
|
||||
pbar.close()
|
||||
|
||||
|
||||
def pt_weights_iterator(
|
||||
|
||||
Reference in New Issue
Block a user