[Feature] Add InstantTensor weight loader (#36139)

This commit is contained in:
arlo
2026-03-14 18:05:23 +01:00
committed by GitHub
parent 5467d137b3
commit 8c29042bb9
13 changed files with 177 additions and 5 deletions
@@ -35,6 +35,7 @@ LoadFormats = Literal[
"dummy",
"fastsafetensors",
"gguf",
"instanttensor",
"mistral",
"npcache",
"pt",
@@ -51,6 +52,7 @@ _LOAD_FORMAT_TO_MODEL_LOADER: dict[str, type[BaseModelLoader]] = {
"dummy": DummyModelLoader,
"fastsafetensors": DefaultModelLoader,
"gguf": GGUFModelLoader,
"instanttensor": DefaultModelLoader,
"mistral": DefaultModelLoader,
"npcache": DefaultModelLoader,
"pt": DefaultModelLoader,
@@ -23,6 +23,7 @@ from vllm.model_executor.model_loader.weight_utils import (
filter_duplicate_safetensors_files,
filter_files_not_needed_for_inference,
get_quant_config,
instanttensor_weights_iterator,
maybe_download_from_modelscope,
multi_thread_pt_weights_iterator,
multi_thread_safetensors_weights_iterator,
@@ -121,7 +122,11 @@ class DefaultModelLoader(BaseModelLoader):
# Some quantized models use .pt files for storing the weights.
if load_format == "hf":
allow_patterns = ["*.safetensors", "*.bin"]
elif load_format == "safetensors" or load_format == "fastsafetensors":
elif (
load_format == "safetensors"
or load_format == "fastsafetensors"
or load_format == "instanttensor"
):
use_safetensors = True
allow_patterns = ["*.safetensors"]
elif load_format == "mistral":
@@ -219,6 +224,11 @@ class DefaultModelLoader(BaseModelLoader):
hf_weights_files,
self.load_config.use_tqdm_on_load,
)
elif self.load_config.load_format == "instanttensor":
weights_iterator = instanttensor_weights_iterator(
hf_weights_files,
self.load_config.use_tqdm_on_load,
)
else:
if extra_config.get("enable_multithread_load"):
weights_iterator = multi_thread_safetensors_weights_iterator(
@@ -29,7 +29,7 @@ from transformers.utils import SAFE_WEIGHTS_INDEX_NAME
from vllm import envs
from vllm.config import ModelConfig
from vllm.config.load import LoadConfig
from vllm.distributed import get_tensor_model_parallel_rank
from vllm.distributed import get_tensor_model_parallel_rank, get_world_group
from vllm.logger import init_logger
from vllm.model_executor.layers.quantization import (
QuantizationConfig,
@@ -909,6 +909,46 @@ def fastsafetensors_weights_iterator(
loader.close()
def instanttensor_weights_iterator(
hf_weights_files: list[str],
use_tqdm_on_load: bool,
) -> Generator[tuple[str, torch.Tensor], None, None]:
"""Iterate over the weights in the model safetensor files
using instanttensor library."""
try:
import instanttensor
except ImportError as e:
raise ImportError(
"Please install instanttensor via `pip install instanttensor`"
) from e
if not current_platform.is_cuda():
raise ValueError("InstantTensor requires NVIDIA GPUs")
try:
world_group = get_world_group()
except AssertionError:
# Entering here only in unit tests where the world group is not initialized.
process_group = None
else:
process_group = world_group.device_group if world_group.world_size > 1 else None
device = current_platform.current_device()
with instanttensor.safe_open(
hf_weights_files, framework="pt", device=device, process_group=process_group
) as f:
yield from tqdm(
f.tensors(),
desc="Loading safetensors using InstantTensor loader",
disable=not enable_tqdm(use_tqdm_on_load),
bar_format=_BAR_FORMAT,
position=tqdm._get_free_pos(),
total=len(f.keys()),
mininterval=1.0,
)
def pt_weights_iterator(
hf_weights_files: list[str],
use_tqdm_on_load: bool,