[Core] Freeze gc during cuda graph capture to speed up init (#21146)
Signed-off-by: Codex <codex@openai.com> Signed-off-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
@@ -2439,10 +2439,25 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
start_time = time.perf_counter()
|
||||
start_free_gpu_memory = torch.cuda.mem_get_info()[0]
|
||||
|
||||
@contextmanager
|
||||
def freeze_gc():
|
||||
# Optimize garbage collection during CUDA graph capture.
|
||||
# Clean up, then freeze all remaining objects from being included
|
||||
# in future collections.
|
||||
gc.collect()
|
||||
should_freeze = not envs.VLLM_ENABLE_CUDAGRAPH_GC
|
||||
if should_freeze:
|
||||
gc.freeze()
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
if should_freeze:
|
||||
gc.unfreeze()
|
||||
|
||||
# Trigger CUDA graph capture for specific shapes.
|
||||
# Capture the large shapes first so that the smaller shapes
|
||||
# can reuse the memory pool allocated for the large shapes.
|
||||
with graph_capture(device=self.device):
|
||||
with freeze_gc(), graph_capture(device=self.device):
|
||||
full_cg = self.full_cuda_graph
|
||||
# Only rank 0 should print progress bar during capture
|
||||
compilation_cases = reversed(self.cudagraph_batch_sizes)
|
||||
|
||||
Reference in New Issue
Block a user