[Feature] Add InstantTensor weight loader (#36139)
This commit is contained in:
@@ -35,6 +35,7 @@ LoadFormats = Literal[
|
||||
"dummy",
|
||||
"fastsafetensors",
|
||||
"gguf",
|
||||
"instanttensor",
|
||||
"mistral",
|
||||
"npcache",
|
||||
"pt",
|
||||
@@ -51,6 +52,7 @@ _LOAD_FORMAT_TO_MODEL_LOADER: dict[str, type[BaseModelLoader]] = {
|
||||
"dummy": DummyModelLoader,
|
||||
"fastsafetensors": DefaultModelLoader,
|
||||
"gguf": GGUFModelLoader,
|
||||
"instanttensor": DefaultModelLoader,
|
||||
"mistral": DefaultModelLoader,
|
||||
"npcache": DefaultModelLoader,
|
||||
"pt": DefaultModelLoader,
|
||||
|
||||
@@ -23,6 +23,7 @@ from vllm.model_executor.model_loader.weight_utils import (
|
||||
filter_duplicate_safetensors_files,
|
||||
filter_files_not_needed_for_inference,
|
||||
get_quant_config,
|
||||
instanttensor_weights_iterator,
|
||||
maybe_download_from_modelscope,
|
||||
multi_thread_pt_weights_iterator,
|
||||
multi_thread_safetensors_weights_iterator,
|
||||
@@ -121,7 +122,11 @@ class DefaultModelLoader(BaseModelLoader):
|
||||
# Some quantized models use .pt files for storing the weights.
|
||||
if load_format == "hf":
|
||||
allow_patterns = ["*.safetensors", "*.bin"]
|
||||
elif load_format == "safetensors" or load_format == "fastsafetensors":
|
||||
elif (
|
||||
load_format == "safetensors"
|
||||
or load_format == "fastsafetensors"
|
||||
or load_format == "instanttensor"
|
||||
):
|
||||
use_safetensors = True
|
||||
allow_patterns = ["*.safetensors"]
|
||||
elif load_format == "mistral":
|
||||
@@ -219,6 +224,11 @@ class DefaultModelLoader(BaseModelLoader):
|
||||
hf_weights_files,
|
||||
self.load_config.use_tqdm_on_load,
|
||||
)
|
||||
elif self.load_config.load_format == "instanttensor":
|
||||
weights_iterator = instanttensor_weights_iterator(
|
||||
hf_weights_files,
|
||||
self.load_config.use_tqdm_on_load,
|
||||
)
|
||||
else:
|
||||
if extra_config.get("enable_multithread_load"):
|
||||
weights_iterator = multi_thread_safetensors_weights_iterator(
|
||||
|
||||
@@ -29,7 +29,7 @@ from transformers.utils import SAFE_WEIGHTS_INDEX_NAME
|
||||
from vllm import envs
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.config.load import LoadConfig
|
||||
from vllm.distributed import get_tensor_model_parallel_rank
|
||||
from vllm.distributed import get_tensor_model_parallel_rank, get_world_group
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.quantization import (
|
||||
QuantizationConfig,
|
||||
@@ -909,6 +909,46 @@ def fastsafetensors_weights_iterator(
|
||||
loader.close()
|
||||
|
||||
|
||||
def instanttensor_weights_iterator(
|
||||
hf_weights_files: list[str],
|
||||
use_tqdm_on_load: bool,
|
||||
) -> Generator[tuple[str, torch.Tensor], None, None]:
|
||||
"""Iterate over the weights in the model safetensor files
|
||||
using instanttensor library."""
|
||||
try:
|
||||
import instanttensor
|
||||
except ImportError as e:
|
||||
raise ImportError(
|
||||
"Please install instanttensor via `pip install instanttensor`"
|
||||
) from e
|
||||
|
||||
if not current_platform.is_cuda():
|
||||
raise ValueError("InstantTensor requires NVIDIA GPUs")
|
||||
|
||||
try:
|
||||
world_group = get_world_group()
|
||||
except AssertionError:
|
||||
# Entering here only in unit tests where the world group is not initialized.
|
||||
process_group = None
|
||||
else:
|
||||
process_group = world_group.device_group if world_group.world_size > 1 else None
|
||||
|
||||
device = current_platform.current_device()
|
||||
|
||||
with instanttensor.safe_open(
|
||||
hf_weights_files, framework="pt", device=device, process_group=process_group
|
||||
) as f:
|
||||
yield from tqdm(
|
||||
f.tensors(),
|
||||
desc="Loading safetensors using InstantTensor loader",
|
||||
disable=not enable_tqdm(use_tqdm_on_load),
|
||||
bar_format=_BAR_FORMAT,
|
||||
position=tqdm._get_free_pos(),
|
||||
total=len(f.keys()),
|
||||
mininterval=1.0,
|
||||
)
|
||||
|
||||
|
||||
def pt_weights_iterator(
|
||||
hf_weights_files: list[str],
|
||||
use_tqdm_on_load: bool,
|
||||
|
||||
Reference in New Issue
Block a user