[Feature] Add InstantTensor weight loader (#36139)
This commit is contained in:
@@ -29,7 +29,7 @@ from transformers.utils import SAFE_WEIGHTS_INDEX_NAME
|
||||
from vllm import envs
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.config.load import LoadConfig
|
||||
from vllm.distributed import get_tensor_model_parallel_rank
|
||||
from vllm.distributed import get_tensor_model_parallel_rank, get_world_group
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.quantization import (
|
||||
QuantizationConfig,
|
||||
@@ -909,6 +909,46 @@ def fastsafetensors_weights_iterator(
|
||||
loader.close()
|
||||
|
||||
|
||||
def instanttensor_weights_iterator(
|
||||
hf_weights_files: list[str],
|
||||
use_tqdm_on_load: bool,
|
||||
) -> Generator[tuple[str, torch.Tensor], None, None]:
|
||||
"""Iterate over the weights in the model safetensor files
|
||||
using instanttensor library."""
|
||||
try:
|
||||
import instanttensor
|
||||
except ImportError as e:
|
||||
raise ImportError(
|
||||
"Please install instanttensor via `pip install instanttensor`"
|
||||
) from e
|
||||
|
||||
if not current_platform.is_cuda():
|
||||
raise ValueError("InstantTensor requires NVIDIA GPUs")
|
||||
|
||||
try:
|
||||
world_group = get_world_group()
|
||||
except AssertionError:
|
||||
# Entering here only in unit tests where the world group is not initialized.
|
||||
process_group = None
|
||||
else:
|
||||
process_group = world_group.device_group if world_group.world_size > 1 else None
|
||||
|
||||
device = current_platform.current_device()
|
||||
|
||||
with instanttensor.safe_open(
|
||||
hf_weights_files, framework="pt", device=device, process_group=process_group
|
||||
) as f:
|
||||
yield from tqdm(
|
||||
f.tensors(),
|
||||
desc="Loading safetensors using InstantTensor loader",
|
||||
disable=not enable_tqdm(use_tqdm_on_load),
|
||||
bar_format=_BAR_FORMAT,
|
||||
position=tqdm._get_free_pos(),
|
||||
total=len(f.keys()),
|
||||
mininterval=1.0,
|
||||
)
|
||||
|
||||
|
||||
def pt_weights_iterator(
|
||||
hf_weights_files: list[str],
|
||||
use_tqdm_on_load: bool,
|
||||
|
||||
Reference in New Issue
Block a user