diff --git a/vllm/model_executor/model_loader/base_loader.py b/vllm/model_executor/model_loader/base_loader.py index 4c774808bf..65f64a3326 100644 --- a/vllm/model_executor/model_loader/base_loader.py +++ b/vllm/model_executor/model_loader/base_loader.py @@ -1,5 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import gc import time from abc import ABC, abstractmethod @@ -86,6 +87,11 @@ class BaseModelLoader(ABC): logger.debug("Loading weights on %s ...", load_device) self.load_weights(model, model_config) + # Release transient tensors held by PyTorch's caching allocator + # before weight post-processing and raw CUDA driver allocations. + gc.collect() + torch.accelerator.empty_cache() + # Log peak GPU memory after loading weights. This is needed # to have test coverage on peak memory for online quantization. if current_platform.is_cuda_alike() or current_platform.is_xpu(): @@ -103,6 +109,9 @@ class BaseModelLoader(ABC): process_weights_after_loading(model, model_config, target_device) log_online_quantization_time(vllm_config) + gc.collect() + torch.accelerator.empty_cache() + return model.eval()