From cc25f028b76844024fddd0b4ca18d4ca169b32be Mon Sep 17 00:00:00 2001 From: Michael Goin Date: Fri, 17 Jul 2026 16:30:02 -0400 Subject: [PATCH] [Loader] Improve InstantTensor loading (#46868) Signed-off-by: mgoin Co-authored-by: OpenAI Codex --- requirements/test/cpu.txt | 2 +- requirements/test/cuda.in | 2 +- requirements/test/cuda.txt | 2 +- requirements/test/nightly-torch.txt | 2 +- requirements/test/rocm.in | 2 +- requirements/test/rocm.txt | 2 +- setup.py | 2 +- .../instanttensor_loader/test_weight_utils.py | 2 -- .../model_loader/weight_utils.py | 25 +++++++++++++++---- 9 files changed, 27 insertions(+), 14 deletions(-) diff --git a/requirements/test/cpu.txt b/requirements/test/cpu.txt index 143155ad7a..d7cec77669 100644 --- a/requirements/test/cpu.txt +++ b/requirements/test/cpu.txt @@ -379,7 +379,7 @@ inflect==5.6.2 # via datamodel-code-generator iniconfig==2.0.0 # via pytest -instanttensor==0.1.5 +instanttensor==0.1.9 # via -r requirements/test/cuda.in interegular==0.3.3 # via lm-format-enforcer diff --git a/requirements/test/cuda.in b/requirements/test/cuda.in index bd6f179c10..d91cba0601 100644 --- a/requirements/test/cuda.in +++ b/requirements/test/cuda.in @@ -58,7 +58,7 @@ arctic-inference == 0.1.1; platform_machine == "x86_64" # Required for suffix de numba == 0.65.0 # Required for N-gram speculative decoding runai-model-streamer[s3,gcs,azure]==0.15.7 fastsafetensors>=0.3.2 -instanttensor>=0.1.5; platform_machine == "x86_64" +instanttensor>=0.1.9; platform_machine == "x86_64" decord==0.6.0; platform_machine == "x86_64" # terratorch is temporarily disabled while PyPI has the `lightning` package # in `quarantined` status (every published terratorch version transitively diff --git a/requirements/test/cuda.txt b/requirements/test/cuda.txt index e4da9ac06e..c4a9ea1ae2 100644 --- a/requirements/test/cuda.txt +++ b/requirements/test/cuda.txt @@ -398,7 +398,7 @@ inflect==5.6.2 # via datamodel-code-generator iniconfig==2.0.0 # via pytest -instanttensor==0.1.5 +instanttensor==0.1.9 # via -r requirements/test/cuda.in interegular==0.3.3 # via lm-format-enforcer diff --git a/requirements/test/nightly-torch.txt b/requirements/test/nightly-torch.txt index dfefa8239c..b308a44013 100644 --- a/requirements/test/nightly-torch.txt +++ b/requirements/test/nightly-torch.txt @@ -44,5 +44,5 @@ numba == 0.65.0 # Required for N-gram speculative decoding numpy runai-model-streamer[s3,gcs,azure]==0.15.7 fastsafetensors>=0.3.2 -instanttensor>=0.1.5 +instanttensor>=0.1.9 pydantic>=2.12 # 2.11 leads to error on python 3.13 diff --git a/requirements/test/rocm.in b/requirements/test/rocm.in index c5f8f85f2f..7cb3b91f7b 100644 --- a/requirements/test/rocm.in +++ b/requirements/test/rocm.in @@ -54,7 +54,7 @@ arctic-inference==0.1.1 # Required for suffix decoding test numba==0.65.0 # Required for N-gram speculative decoding runai-model-streamer[s3,gcs,azure]==0.15.7 fastsafetensors>=0.3.2 -instanttensor>=0.1.5 +instanttensor>=0.1.9 decord==0.6.0 # Prithvi tests diff --git a/requirements/test/rocm.txt b/requirements/test/rocm.txt index 6cd27abe17..f274a93362 100644 --- a/requirements/test/rocm.txt +++ b/requirements/test/rocm.txt @@ -391,7 +391,7 @@ inflect==7.5.0 # via datamodel-code-generator iniconfig==2.3.0 # via pytest -instanttensor==0.1.6 +instanttensor==0.1.9 # via -r requirements/test/rocm.in interegular==0.3.3 # via lm-format-enforcer diff --git a/setup.py b/setup.py index a868515793..40d4ca103b 100644 --- a/setup.py +++ b/setup.py @@ -1268,7 +1268,7 @@ setup( "bench": ["pandas", "matplotlib", "seaborn", "datasets", "scipy", "plotly"], "tensorizer": ["tensorizer==2.10.1"], "fastsafetensors": ["fastsafetensors >= 0.3.2"], - "instanttensor": ["instanttensor >= 0.1.5"], + "instanttensor": ["instanttensor >= 0.1.9"], "runai": ["runai-model-streamer[s3,gcs,azure] >= 0.15.7"], "audio": [ "av", diff --git a/tests/model_executor/model_loader/instanttensor_loader/test_weight_utils.py b/tests/model_executor/model_loader/instanttensor_loader/test_weight_utils.py index 992a83e0ee..09e0a19d11 100644 --- a/tests/model_executor/model_loader/instanttensor_loader/test_weight_utils.py +++ b/tests/model_executor/model_loader/instanttensor_loader/test_weight_utils.py @@ -33,8 +33,6 @@ def test_instanttensor_model_loader(): hf_safetensors_tensors = {} for name, tensor in instanttensor_weights_iterator(safetensors, True): - # Copy the tensor immediately as it is a reference to the internal - # buffer of instanttensor. instanttensor_tensors[name] = tensor.to("cpu") for name, tensor in safetensors_weights_iterator(safetensors, True): diff --git a/vllm/model_executor/model_loader/weight_utils.py b/vllm/model_executor/model_loader/weight_utils.py index 8e673ae7fc..db161a5898 100644 --- a/vllm/model_executor/model_loader/weight_utils.py +++ b/vllm/model_executor/model_loader/weight_utils.py @@ -1120,7 +1120,7 @@ def instanttensor_weights_iterator( import instanttensor except ImportError as e: raise ImportError( - "Please install instanttensor via `pip install instanttensor`" + "Please install instanttensor via `pip install vllm[instanttensor]`" ) from e if not current_platform.is_cuda(): @@ -1136,18 +1136,33 @@ def instanttensor_weights_iterator( device = current_platform.current_device() + # copy=True yields tensors that own their memory, staying valid after the + # context exits or InstantTensor reuses its buffer. with instanttensor.safe_open( - hf_weights_files, framework="pt", device=device, process_group=process_group + hf_weights_files, + framework="pt", + device=device, + process_group=process_group, + copy=True, ) as f: - yield from tqdm( - f.tensors(), + # Track bytes so the bar reports load throughput (GB/s). + pbar = tqdm( + total=f.total_tensor_size, desc="Loading safetensors using InstantTensor loader", disable=not enable_tqdm(use_tqdm_on_load), bar_format=_BAR_FORMAT, position=tqdm._get_free_pos(), - total=len(f.keys()), + unit="B", + unit_scale=True, + unit_divisor=1024, mininterval=1.0, ) + try: + for name, tensor in f.tensors(): + pbar.update(tensor.numel() * tensor.element_size()) + yield name, tensor + finally: + pbar.close() def pt_weights_iterator(