[Core] Use fastsafetensors ParallelLoader for weight loading (#40183)

Signed-off-by: Git Bisector <gitbisector@gmail.com>
Signed-off-by: gitbisector <gitbisector@gmail.com>
Signed-off-by: git bisector <gitbisector@gmail.com>
Co-authored-by: Claude <noreply@anthropic.com>
Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
gitbisector
2026-06-15 22:32:05 -07:00
committed by GitHub
co-authored by Claude Cyrus Leung
parent f3858d5422
commit 9d808e2309
3 changed files with 68 additions and 56 deletions
@@ -20,7 +20,9 @@ from vllm.platforms import current_platform
not current_platform.is_cuda_alike(),
reason="fastsafetensors requires NVIDIA/AMD GPUs",
)
def test_fastsafetensors_model_loader():
@pytest.mark.parametrize("queue_size", [0, 1])
def test_fastsafetensors_model_loader(monkeypatch, queue_size):
monkeypatch.setenv("VLLM_FASTSAFETENSORS_QUEUE_SIZE", str(queue_size))
with tempfile.TemporaryDirectory() as tmpdir:
huggingface_hub.constants.HF_HUB_OFFLINE = False
download_weights_from_hf(
@@ -45,7 +47,3 @@ def test_fastsafetensors_model_loader():
assert fastsafetensors_tensor.dtype == hf_safetensors_tensors[name].dtype
assert fastsafetensors_tensor.shape == hf_safetensors_tensors[name].shape
assert torch.all(fastsafetensors_tensor.eq(hf_safetensors_tensors[name]))
if __name__ == "__main__":
test_fastsafetensors_model_loader()