[Feature] Add InstantTensor weight loader (#36139)

This commit is contained in:
arlo
2026-03-14 18:05:23 +01:00
committed by GitHub
parent 5467d137b3
commit 8c29042bb9
13 changed files with 177 additions and 5 deletions
@@ -23,6 +23,7 @@ from vllm.model_executor.model_loader.weight_utils import (
filter_duplicate_safetensors_files,
filter_files_not_needed_for_inference,
get_quant_config,
instanttensor_weights_iterator,
maybe_download_from_modelscope,
multi_thread_pt_weights_iterator,
multi_thread_safetensors_weights_iterator,
@@ -121,7 +122,11 @@ class DefaultModelLoader(BaseModelLoader):
# Some quantized models use .pt files for storing the weights.
if load_format == "hf":
allow_patterns = ["*.safetensors", "*.bin"]
elif load_format == "safetensors" or load_format == "fastsafetensors":
elif (
load_format == "safetensors"
or load_format == "fastsafetensors"
or load_format == "instanttensor"
):
use_safetensors = True
allow_patterns = ["*.safetensors"]
elif load_format == "mistral":
@@ -219,6 +224,11 @@ class DefaultModelLoader(BaseModelLoader):
hf_weights_files,
self.load_config.use_tqdm_on_load,
)
elif self.load_config.load_format == "instanttensor":
weights_iterator = instanttensor_weights_iterator(
hf_weights_files,
self.load_config.use_tqdm_on_load,
)
else:
if extra_config.get("enable_multithread_load"):
weights_iterator = multi_thread_safetensors_weights_iterator(