[Core] Use fastsafetensors ParallelLoader for weight loading (#40183)
Signed-off-by: Git Bisector <gitbisector@gmail.com> Signed-off-by: gitbisector <gitbisector@gmail.com> Signed-off-by: git bisector <gitbisector@gmail.com> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
co-authored by
Claude
Cyrus Leung
parent
f3858d5422
commit
9d808e2309
@@ -20,7 +20,9 @@ from vllm.platforms import current_platform
|
||||
not current_platform.is_cuda_alike(),
|
||||
reason="fastsafetensors requires NVIDIA/AMD GPUs",
|
||||
)
|
||||
def test_fastsafetensors_model_loader():
|
||||
@pytest.mark.parametrize("queue_size", [0, 1])
|
||||
def test_fastsafetensors_model_loader(monkeypatch, queue_size):
|
||||
monkeypatch.setenv("VLLM_FASTSAFETENSORS_QUEUE_SIZE", str(queue_size))
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
huggingface_hub.constants.HF_HUB_OFFLINE = False
|
||||
download_weights_from_hf(
|
||||
@@ -45,7 +47,3 @@ def test_fastsafetensors_model_loader():
|
||||
assert fastsafetensors_tensor.dtype == hf_safetensors_tensors[name].dtype
|
||||
assert fastsafetensors_tensor.shape == hf_safetensors_tensors[name].shape
|
||||
assert torch.all(fastsafetensors_tensor.eq(hf_safetensors_tensors[name]))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
test_fastsafetensors_model_loader()
|
||||
|
||||
Reference in New Issue
Block a user