[Bugfix] Reclaim cached GPU memory during weight loading (#57516)
Signed-off-by: Debanshu Das <debanshu@google.com> Signed-off-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Kimi Code <noreply@moonshot.cn>
This commit is contained in:
co-authored by
Isotr0py
Kimi Code
parent
61f084b5f3
commit
ef42b4385f
@@ -1,5 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import gc
|
||||
import time
|
||||
from abc import ABC, abstractmethod
|
||||
|
||||
@@ -86,6 +87,11 @@ class BaseModelLoader(ABC):
|
||||
logger.debug("Loading weights on %s ...", load_device)
|
||||
self.load_weights(model, model_config)
|
||||
|
||||
# Release transient tensors held by PyTorch's caching allocator
|
||||
# before weight post-processing and raw CUDA driver allocations.
|
||||
gc.collect()
|
||||
torch.accelerator.empty_cache()
|
||||
|
||||
# Log peak GPU memory after loading weights. This is needed
|
||||
# to have test coverage on peak memory for online quantization.
|
||||
if current_platform.is_cuda_alike() or current_platform.is_xpu():
|
||||
@@ -103,6 +109,9 @@ class BaseModelLoader(ABC):
|
||||
process_weights_after_loading(model, model_config, target_device)
|
||||
log_online_quantization_time(vllm_config)
|
||||
|
||||
gc.collect()
|
||||
torch.accelerator.empty_cache()
|
||||
|
||||
return model.eval()
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user