[Loader] Improve InstantTensor loading (#46868)

Signed-off-by: mgoin <mgoin64@gmail.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
Michael Goin
2026-07-17 16:30:02 -04:00
committed by GitHub
co-authored by OpenAI Codex
parent c4cd2bd544
commit cc25f028b7
9 changed files with 27 additions and 14 deletions
@@ -1120,7 +1120,7 @@ def instanttensor_weights_iterator(
import instanttensor
except ImportError as e:
raise ImportError(
"Please install instanttensor via `pip install instanttensor`"
"Please install instanttensor via `pip install vllm[instanttensor]`"
) from e
if not current_platform.is_cuda():
@@ -1136,18 +1136,33 @@ def instanttensor_weights_iterator(
device = current_platform.current_device()
# copy=True yields tensors that own their memory, staying valid after the
# context exits or InstantTensor reuses its buffer.
with instanttensor.safe_open(
hf_weights_files, framework="pt", device=device, process_group=process_group
hf_weights_files,
framework="pt",
device=device,
process_group=process_group,
copy=True,
) as f:
yield from tqdm(
f.tensors(),
# Track bytes so the bar reports load throughput (GB/s).
pbar = tqdm(
total=f.total_tensor_size,
desc="Loading safetensors using InstantTensor loader",
disable=not enable_tqdm(use_tqdm_on_load),
bar_format=_BAR_FORMAT,
position=tqdm._get_free_pos(),
total=len(f.keys()),
unit="B",
unit_scale=True,
unit_divisor=1024,
mininterval=1.0,
)
try:
for name, tensor in f.tensors():
pbar.update(tensor.numel() * tensor.element_size())
yield name, tensor
finally:
pbar.close()
def pt_weights_iterator(