[Feature] Add InstantTensor weight loader (#36139)

This commit is contained in:
arlo
2026-03-14 18:05:23 +01:00
committed by GitHub
parent 5467d137b3
commit 8c29042bb9
13 changed files with 177 additions and 5 deletions
@@ -29,7 +29,7 @@ from transformers.utils import SAFE_WEIGHTS_INDEX_NAME
from vllm import envs
from vllm.config import ModelConfig
from vllm.config.load import LoadConfig
from vllm.distributed import get_tensor_model_parallel_rank
from vllm.distributed import get_tensor_model_parallel_rank, get_world_group
from vllm.logger import init_logger
from vllm.model_executor.layers.quantization import (
QuantizationConfig,
@@ -909,6 +909,46 @@ def fastsafetensors_weights_iterator(
loader.close()
def instanttensor_weights_iterator(
hf_weights_files: list[str],
use_tqdm_on_load: bool,
) -> Generator[tuple[str, torch.Tensor], None, None]:
"""Iterate over the weights in the model safetensor files
using instanttensor library."""
try:
import instanttensor
except ImportError as e:
raise ImportError(
"Please install instanttensor via `pip install instanttensor`"
) from e
if not current_platform.is_cuda():
raise ValueError("InstantTensor requires NVIDIA GPUs")
try:
world_group = get_world_group()
except AssertionError:
# Entering here only in unit tests where the world group is not initialized.
process_group = None
else:
process_group = world_group.device_group if world_group.world_size > 1 else None
device = current_platform.current_device()
with instanttensor.safe_open(
hf_weights_files, framework="pt", device=device, process_group=process_group
) as f:
yield from tqdm(
f.tensors(),
desc="Loading safetensors using InstantTensor loader",
disable=not enable_tqdm(use_tqdm_on_load),
bar_format=_BAR_FORMAT,
position=tqdm._get_free_pos(),
total=len(f.keys()),
mininterval=1.0,
)
def pt_weights_iterator(
hf_weights_files: list[str],
use_tqdm_on_load: bool,