[Loader] Improve InstantTensor loading (#46868)
Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
c4cd2bd544
commit
cc25f028b7
@@ -379,7 +379,7 @@ inflect==5.6.2
|
||||
# via datamodel-code-generator
|
||||
iniconfig==2.0.0
|
||||
# via pytest
|
||||
instanttensor==0.1.5
|
||||
instanttensor==0.1.9
|
||||
# via -r requirements/test/cuda.in
|
||||
interegular==0.3.3
|
||||
# via lm-format-enforcer
|
||||
|
||||
@@ -58,7 +58,7 @@ arctic-inference == 0.1.1; platform_machine == "x86_64" # Required for suffix de
|
||||
numba == 0.65.0 # Required for N-gram speculative decoding
|
||||
runai-model-streamer[s3,gcs,azure]==0.15.7
|
||||
fastsafetensors>=0.3.2
|
||||
instanttensor>=0.1.5; platform_machine == "x86_64"
|
||||
instanttensor>=0.1.9; platform_machine == "x86_64"
|
||||
decord==0.6.0; platform_machine == "x86_64"
|
||||
# terratorch is temporarily disabled while PyPI has the `lightning` package
|
||||
# in `quarantined` status (every published terratorch version transitively
|
||||
|
||||
@@ -398,7 +398,7 @@ inflect==5.6.2
|
||||
# via datamodel-code-generator
|
||||
iniconfig==2.0.0
|
||||
# via pytest
|
||||
instanttensor==0.1.5
|
||||
instanttensor==0.1.9
|
||||
# via -r requirements/test/cuda.in
|
||||
interegular==0.3.3
|
||||
# via lm-format-enforcer
|
||||
|
||||
@@ -44,5 +44,5 @@ numba == 0.65.0 # Required for N-gram speculative decoding
|
||||
numpy
|
||||
runai-model-streamer[s3,gcs,azure]==0.15.7
|
||||
fastsafetensors>=0.3.2
|
||||
instanttensor>=0.1.5
|
||||
instanttensor>=0.1.9
|
||||
pydantic>=2.12 # 2.11 leads to error on python 3.13
|
||||
|
||||
@@ -54,7 +54,7 @@ arctic-inference==0.1.1 # Required for suffix decoding test
|
||||
numba==0.65.0 # Required for N-gram speculative decoding
|
||||
runai-model-streamer[s3,gcs,azure]==0.15.7
|
||||
fastsafetensors>=0.3.2
|
||||
instanttensor>=0.1.5
|
||||
instanttensor>=0.1.9
|
||||
decord==0.6.0
|
||||
|
||||
# Prithvi tests
|
||||
|
||||
@@ -391,7 +391,7 @@ inflect==7.5.0
|
||||
# via datamodel-code-generator
|
||||
iniconfig==2.3.0
|
||||
# via pytest
|
||||
instanttensor==0.1.6
|
||||
instanttensor==0.1.9
|
||||
# via -r requirements/test/rocm.in
|
||||
interegular==0.3.3
|
||||
# via lm-format-enforcer
|
||||
|
||||
@@ -1268,7 +1268,7 @@ setup(
|
||||
"bench": ["pandas", "matplotlib", "seaborn", "datasets", "scipy", "plotly"],
|
||||
"tensorizer": ["tensorizer==2.10.1"],
|
||||
"fastsafetensors": ["fastsafetensors >= 0.3.2"],
|
||||
"instanttensor": ["instanttensor >= 0.1.5"],
|
||||
"instanttensor": ["instanttensor >= 0.1.9"],
|
||||
"runai": ["runai-model-streamer[s3,gcs,azure] >= 0.15.7"],
|
||||
"audio": [
|
||||
"av",
|
||||
|
||||
@@ -33,8 +33,6 @@ def test_instanttensor_model_loader():
|
||||
hf_safetensors_tensors = {}
|
||||
|
||||
for name, tensor in instanttensor_weights_iterator(safetensors, True):
|
||||
# Copy the tensor immediately as it is a reference to the internal
|
||||
# buffer of instanttensor.
|
||||
instanttensor_tensors[name] = tensor.to("cpu")
|
||||
|
||||
for name, tensor in safetensors_weights_iterator(safetensors, True):
|
||||
|
||||
@@ -1120,7 +1120,7 @@ def instanttensor_weights_iterator(
|
||||
import instanttensor
|
||||
except ImportError as e:
|
||||
raise ImportError(
|
||||
"Please install instanttensor via `pip install instanttensor`"
|
||||
"Please install instanttensor via `pip install vllm[instanttensor]`"
|
||||
) from e
|
||||
|
||||
if not current_platform.is_cuda():
|
||||
@@ -1136,18 +1136,33 @@ def instanttensor_weights_iterator(
|
||||
|
||||
device = current_platform.current_device()
|
||||
|
||||
# copy=True yields tensors that own their memory, staying valid after the
|
||||
# context exits or InstantTensor reuses its buffer.
|
||||
with instanttensor.safe_open(
|
||||
hf_weights_files, framework="pt", device=device, process_group=process_group
|
||||
hf_weights_files,
|
||||
framework="pt",
|
||||
device=device,
|
||||
process_group=process_group,
|
||||
copy=True,
|
||||
) as f:
|
||||
yield from tqdm(
|
||||
f.tensors(),
|
||||
# Track bytes so the bar reports load throughput (GB/s).
|
||||
pbar = tqdm(
|
||||
total=f.total_tensor_size,
|
||||
desc="Loading safetensors using InstantTensor loader",
|
||||
disable=not enable_tqdm(use_tqdm_on_load),
|
||||
bar_format=_BAR_FORMAT,
|
||||
position=tqdm._get_free_pos(),
|
||||
total=len(f.keys()),
|
||||
unit="B",
|
||||
unit_scale=True,
|
||||
unit_divisor=1024,
|
||||
mininterval=1.0,
|
||||
)
|
||||
try:
|
||||
for name, tensor in f.tensors():
|
||||
pbar.update(tensor.numel() * tensor.element_size())
|
||||
yield name, tensor
|
||||
finally:
|
||||
pbar.close()
|
||||
|
||||
|
||||
def pt_weights_iterator(
|
||||
|
||||
Reference in New Issue
Block a user