[Core] Freeze garbage collector in workers after model initialization (#44363)
Signed-off-by: Tyler Michael Smith <tlrmchlsmth@gmail.com>
This commit is contained in:
@@ -51,6 +51,7 @@ from vllm.profiler.wrapper import CudaProfilerWrapper, TorchProfilerWrapper
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.tasks import SupportedTask
|
||||
from vllm.tracing import instrument
|
||||
from vllm.utils.gc_utils import freeze_gc_heap, maybe_attach_gc_debug_callback
|
||||
from vllm.utils.mem_constants import GiB_bytes
|
||||
from vllm.utils.mem_utils import MemorySnapshot, format_gib, memory_profiling
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
@@ -722,6 +723,11 @@ class Worker(WorkerBase):
|
||||
|
||||
activate_triton_jit_monitor()
|
||||
|
||||
# Freeze the worker heap so the GC won't scan static objects
|
||||
# (model weights, KV caches, CUDA graphs) during inference.
|
||||
freeze_gc_heap()
|
||||
maybe_attach_gc_debug_callback()
|
||||
|
||||
return CompilationTimes(
|
||||
language_model=self.compilation_config.compilation_time,
|
||||
encoder=self.compilation_config.encoder_compilation_time,
|
||||
@@ -1115,6 +1121,8 @@ class Worker(WorkerBase):
|
||||
self._is_checkpoint_format = True
|
||||
|
||||
def shutdown(self) -> None:
|
||||
gc.unfreeze()
|
||||
|
||||
# has_kv_transfer_group can be None during interpreter shutdown.
|
||||
if ensure_kv_transfer_shutdown is not None:
|
||||
ensure_kv_transfer_shutdown()
|
||||
|
||||
Reference in New Issue
Block a user