[Core] Release cached device memory under pressure on UMA GPUs during weight loading (#45179)

Signed-off-by: mgoin <mgoin64@gmail.com>
Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
Michael Goin
2026-06-11 17:49:50 +03:00
committed by GitHub
co-authored by Claude
parent b78fc47f05
commit 5edf7ff489
2 changed files with 42 additions and 0 deletions
@@ -30,6 +30,7 @@ from vllm.model_executor.model_loader.reload import (
)
from vllm.model_executor.models.interfaces import SupportsQuant
from vllm.tracing import instrument
from vllm.utils.mem_utils import release_device_memory_under_pressure
from vllm.utils.platform_utils import is_pin_memory_available
from vllm.utils.torch_utils import get_accelerator_view_from_cpu_tensor
@@ -109,6 +110,9 @@ def process_weights_after_loading(
# parameters onto device for processing and back off after.
with device_loading_context(module, target_device):
quant_method.process_weights_after_loading(module)
# Repacking transients above can leave large amounts of memory in
# the caching allocator, which starves the OS on UMA devices.
release_device_memory_under_pressure(target_device)
# Initialize post-load attention weights for Attention, MLA, and MM encoder.
# NOTE: Happens after other modules so we can easily decompress weights.