[Bugfix] Reclaim cached GPU memory during weight loading (#57516)

Signed-off-by: Debanshu Das <debanshu@google.com>
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: Kimi Code <noreply@moonshot.cn>
This commit is contained in:
debanshd
2026-10-11 07:03:06 +00:00
committed by GitHub
co-authored by Isotr0py Kimi Code
parent 61f084b5f3
commit ef42b4385f
@@ -1,5 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import gc
import time
from abc import ABC, abstractmethod
@@ -86,6 +87,11 @@ class BaseModelLoader(ABC):
logger.debug("Loading weights on %s ...", load_device)
self.load_weights(model, model_config)
# Release transient tensors held by PyTorch's caching allocator
# before weight post-processing and raw CUDA driver allocations.
gc.collect()
torch.accelerator.empty_cache()
# Log peak GPU memory after loading weights. This is needed
# to have test coverage on peak memory for online quantization.
if current_platform.is_cuda_alike() or current_platform.is_xpu():
@@ -103,6 +109,9 @@ class BaseModelLoader(ABC):
process_weights_after_loading(model, model_config, target_device)
log_online_quantization_time(vllm_config)
gc.collect()
torch.accelerator.empty_cache()
return model.eval()