[Loader] Improve InstantTensor loading (#46868)

Signed-off-by: mgoin <mgoin64@gmail.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
Michael Goin
2026-07-17 16:30:02 -04:00
committed by GitHub
co-authored by OpenAI Codex
parent c4cd2bd544
commit cc25f028b7
9 changed files with 27 additions and 14 deletions
+1 -1
View File
@@ -379,7 +379,7 @@ inflect==5.6.2
# via datamodel-code-generator
iniconfig==2.0.0
# via pytest
instanttensor==0.1.5
instanttensor==0.1.9
# via -r requirements/test/cuda.in
interegular==0.3.3
# via lm-format-enforcer
+1 -1
View File
@@ -58,7 +58,7 @@ arctic-inference == 0.1.1; platform_machine == "x86_64" # Required for suffix de
numba == 0.65.0 # Required for N-gram speculative decoding
runai-model-streamer[s3,gcs,azure]==0.15.7
fastsafetensors>=0.3.2
instanttensor>=0.1.5; platform_machine == "x86_64"
instanttensor>=0.1.9; platform_machine == "x86_64"
decord==0.6.0; platform_machine == "x86_64"
# terratorch is temporarily disabled while PyPI has the `lightning` package
# in `quarantined` status (every published terratorch version transitively
+1 -1
View File
@@ -398,7 +398,7 @@ inflect==5.6.2
# via datamodel-code-generator
iniconfig==2.0.0
# via pytest
instanttensor==0.1.5
instanttensor==0.1.9
# via -r requirements/test/cuda.in
interegular==0.3.3
# via lm-format-enforcer
+1 -1
View File
@@ -44,5 +44,5 @@ numba == 0.65.0 # Required for N-gram speculative decoding
numpy
runai-model-streamer[s3,gcs,azure]==0.15.7
fastsafetensors>=0.3.2
instanttensor>=0.1.5
instanttensor>=0.1.9
pydantic>=2.12 # 2.11 leads to error on python 3.13
+1 -1
View File
@@ -54,7 +54,7 @@ arctic-inference==0.1.1 # Required for suffix decoding test
numba==0.65.0 # Required for N-gram speculative decoding
runai-model-streamer[s3,gcs,azure]==0.15.7
fastsafetensors>=0.3.2
instanttensor>=0.1.5
instanttensor>=0.1.9
decord==0.6.0
# Prithvi tests
+1 -1
View File
@@ -391,7 +391,7 @@ inflect==7.5.0
# via datamodel-code-generator
iniconfig==2.3.0
# via pytest
instanttensor==0.1.6
instanttensor==0.1.9
# via -r requirements/test/rocm.in
interegular==0.3.3
# via lm-format-enforcer
+1 -1
View File
@@ -1268,7 +1268,7 @@ setup(
"bench": ["pandas", "matplotlib", "seaborn", "datasets", "scipy", "plotly"],
"tensorizer": ["tensorizer==2.10.1"],
"fastsafetensors": ["fastsafetensors >= 0.3.2"],
"instanttensor": ["instanttensor >= 0.1.5"],
"instanttensor": ["instanttensor >= 0.1.9"],
"runai": ["runai-model-streamer[s3,gcs,azure] >= 0.15.7"],
"audio": [
"av",
@@ -33,8 +33,6 @@ def test_instanttensor_model_loader():
hf_safetensors_tensors = {}
for name, tensor in instanttensor_weights_iterator(safetensors, True):
# Copy the tensor immediately as it is a reference to the internal
# buffer of instanttensor.
instanttensor_tensors[name] = tensor.to("cpu")
for name, tensor in safetensors_weights_iterator(safetensors, True):
@@ -1120,7 +1120,7 @@ def instanttensor_weights_iterator(
import instanttensor
except ImportError as e:
raise ImportError(
"Please install instanttensor via `pip install instanttensor`"
"Please install instanttensor via `pip install vllm[instanttensor]`"
) from e
if not current_platform.is_cuda():
@@ -1136,18 +1136,33 @@ def instanttensor_weights_iterator(
device = current_platform.current_device()
# copy=True yields tensors that own their memory, staying valid after the
# context exits or InstantTensor reuses its buffer.
with instanttensor.safe_open(
hf_weights_files, framework="pt", device=device, process_group=process_group
hf_weights_files,
framework="pt",
device=device,
process_group=process_group,
copy=True,
) as f:
yield from tqdm(
f.tensors(),
# Track bytes so the bar reports load throughput (GB/s).
pbar = tqdm(
total=f.total_tensor_size,
desc="Loading safetensors using InstantTensor loader",
disable=not enable_tqdm(use_tqdm_on_load),
bar_format=_BAR_FORMAT,
position=tqdm._get_free_pos(),
total=len(f.keys()),
unit="B",
unit_scale=True,
unit_divisor=1024,
mininterval=1.0,
)
try:
for name, tensor in f.tensors():
pbar.update(tensor.numel() * tensor.element_size())
yield name, tensor
finally:
pbar.close()
def pt_weights_iterator(