diff --git a/vllm/v1/worker/gpu_worker.py b/vllm/v1/worker/gpu_worker.py index 259cd05554..1b30a981e2 100644 --- a/vllm/v1/worker/gpu_worker.py +++ b/vllm/v1/worker/gpu_worker.py @@ -51,6 +51,7 @@ from vllm.profiler.wrapper import CudaProfilerWrapper, TorchProfilerWrapper from vllm.sequence import IntermediateTensors from vllm.tasks import SupportedTask from vllm.tracing import instrument +from vllm.utils.gc_utils import freeze_gc_heap, maybe_attach_gc_debug_callback from vllm.utils.mem_constants import GiB_bytes from vllm.utils.mem_utils import MemorySnapshot, format_gib, memory_profiling from vllm.utils.torch_utils import set_random_seed @@ -722,6 +723,11 @@ class Worker(WorkerBase): activate_triton_jit_monitor() + # Freeze the worker heap so the GC won't scan static objects + # (model weights, KV caches, CUDA graphs) during inference. + freeze_gc_heap() + maybe_attach_gc_debug_callback() + return CompilationTimes( language_model=self.compilation_config.compilation_time, encoder=self.compilation_config.encoder_compilation_time, @@ -1115,6 +1121,8 @@ class Worker(WorkerBase): self._is_checkpoint_format = True def shutdown(self) -> None: + gc.unfreeze() + # has_kv_transfer_group can be None during interpreter shutdown. if ensure_kv_transfer_shutdown is not None: ensure_kv_transfer_shutdown()