[Core] Release cached device memory under pressure on UMA GPUs during weight loading (#45179)
Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -30,6 +30,7 @@ from vllm.model_executor.model_loader.reload import (
|
||||
)
|
||||
from vllm.model_executor.models.interfaces import SupportsQuant
|
||||
from vllm.tracing import instrument
|
||||
from vllm.utils.mem_utils import release_device_memory_under_pressure
|
||||
from vllm.utils.platform_utils import is_pin_memory_available
|
||||
from vllm.utils.torch_utils import get_accelerator_view_from_cpu_tensor
|
||||
|
||||
@@ -109,6 +110,9 @@ def process_weights_after_loading(
|
||||
# parameters onto device for processing and back off after.
|
||||
with device_loading_context(module, target_device):
|
||||
quant_method.process_weights_after_loading(module)
|
||||
# Repacking transients above can leave large amounts of memory in
|
||||
# the caching allocator, which starves the OS on UMA devices.
|
||||
release_device_memory_under_pressure(target_device)
|
||||
|
||||
# Initialize post-load attention weights for Attention, MLA, and MM encoder.
|
||||
# NOTE: Happens after other modules so we can easily decompress weights.
|
||||
|
||||
Reference in New Issue
Block a user