[Core][MRV2] Freeze gc during V2 CG capture; skip per-descriptor cleanup (#54646)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Nick Hill
2026-08-31 19:37:46 -07:00
committed by GitHub
parent 225aec4809
commit c28feab989
4 changed files with 76 additions and 67 deletions
+7 -9
View File
@@ -365,15 +365,13 @@ class BreakableCUDAGraphWrapper:
else:
set_graph_pool_id(current_platform.graph_pool_handle())
# Match torch.cuda.graph()'s pre-capture cleanup once per descriptor.
# We drive capture_begin/end directly and bypass torch.cuda.graph(),
# so its built-in gc + empty_cache never fire. Run them here once
# per _capture call -- NOT inside _begin_segment, since this capture
# session may issue many begin/end pairs (one per layer's break),
# and repeated gc would tank capture time the way it did for the
# pre-`gc_disable` piecewise path.
gc.collect()
torch.accelerator.empty_cache()
# Match torch.cuda.graph()'s pre-capture cleanup, which we bypass.
# Skip it when gc is disabled: bulk capture runs under
# freeze_gc_for_cudagraph_capture, which already did this cleanup,
# and repeating it per descriptor dominates capture time.
if gc.isenabled():
gc.collect()
torch.accelerator.empty_cache()
# Sync the offloader's copy stream before capture so any in-flight
# pre-capture prefetches are complete and don't leak into the graph.
get_offloader().sync_prev_onload()
+29 -1
View File
@@ -4,7 +4,7 @@ import gc
import json
import time
from collections import Counter
from contextlib import suppress
from contextlib import contextmanager, suppress
from typing import Any
import vllm.envs as envs
@@ -93,6 +93,34 @@ class GCDebugger:
)
@contextmanager
def freeze_gc_for_cudagraph_capture():
"""Freeze and disable gc for the duration of bulk CUDA graph capture.
A gc cycle during stream capture can invalidate the captured graph, e.g.
a finalized Triton kernel unloads its module. Opt out with
VLLM_ENABLE_CUDAGRAPH_GC=1.
"""
gc_was_enabled = gc.isenabled()
gc.collect()
should_freeze = not envs.VLLM_ENABLE_CUDAGRAPH_GC
if should_freeze:
gc.freeze()
gc.disable()
try:
yield
finally:
if should_freeze:
try:
gc.unfreeze()
gc.collect()
finally:
if gc_was_enabled:
gc.enable()
else:
gc.disable()
def freeze_gc_heap() -> None:
"""
Freeze all objects tracked by the garbage collector. It should be invoked
+38 -34
View File
@@ -61,6 +61,7 @@ from vllm.multimodal.encoder_budget import (
)
from vllm.sequence import IntermediateTensors
from vllm.tasks import SupportedTask
from vllm.utils.gc_utils import freeze_gc_for_cudagraph_capture
from vllm.utils.mem_utils import DeviceMemoryProfiler, format_gib
from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE
from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput
@@ -916,44 +917,47 @@ class GPUModelRunner(LoRAModelRunnerMixin):
compilation_counter.num_gpu_runner_capture_triggers += 1
start_time = time.perf_counter()
gc.collect()
torch.accelerator.empty_cache()
start_free_gpu_memory = torch.accelerator.get_memory_info()[0]
with freeze_gc_for_cudagraph_capture():
torch.accelerator.empty_cache()
start_free_gpu_memory = torch.accelerator.get_memory_info()[0]
with self.maybe_setup_dummy_loras(self.lora_config):
if capture_encoder:
self.model_state.encoder_runner.capture()
with self.maybe_setup_dummy_loras(self.lora_config):
if capture_encoder:
self.model_state.encoder_runner.capture()
if capture_decoder:
input_buffers = self.input_buffers
if self.pcp_manager is not None:
input_buffers = self.pcp_manager.input_buffers
self.cudagraph_manager.capture(
self.model,
self.model_state,
input_buffers,
self.intermediate_tensors,
self.block_tables,
self.attn_groups,
self.kv_cache_config,
pcp_manager=self.pcp_manager,
has_lora=self.lora_config is not None,
use_aux_hidden_state_outputs=self.use_aux_hidden_state_outputs,
lora_capture_hook=create_lora_capture_hook(self.lora_config, self),
)
if self.speculator is not None:
with use_workspace_lane(self._draft_workspace_lane):
self.speculator.capture()
if self.adaptive_verification is not None:
with self.step_timing.collect() as timings:
for batch in self.adaptive_verification.batches_to_profile(
self.cudagraph_manager.captured_token_counts()
):
self._dummy_run(**batch)
self.adaptive_verification.set_initial_cost_curves(timings)
if capture_decoder:
input_buffers = self.input_buffers
if self.pcp_manager is not None:
input_buffers = self.pcp_manager.input_buffers
self.cudagraph_manager.capture(
self.model,
self.model_state,
input_buffers,
self.intermediate_tensors,
self.block_tables,
self.attn_groups,
self.kv_cache_config,
pcp_manager=self.pcp_manager,
has_lora=self.lora_config is not None,
use_aux_hidden_state_outputs=self.use_aux_hidden_state_outputs,
lora_capture_hook=create_lora_capture_hook(
self.lora_config, self
),
)
if self.speculator is not None:
with use_workspace_lane(self._draft_workspace_lane):
self.speculator.capture()
if self.adaptive_verification is not None:
with self.step_timing.collect() as timings:
for batch in self.adaptive_verification.batches_to_profile(
self.cudagraph_manager.captured_token_counts()
):
self._dummy_run(**batch)
self.adaptive_verification.set_initial_cost_curves(timings)
end_free_gpu_memory = torch.accelerator.get_memory_info()[0]
end_time = time.perf_counter()
end_free_gpu_memory = torch.accelerator.get_memory_info()[0]
elapsed_time = end_time - start_time
cuda_graph_size = start_free_gpu_memory - end_free_gpu_memory
# This usually takes 5~20 seconds.
+2 -23
View File
@@ -124,6 +124,7 @@ from vllm.sequence import IntermediateTensors
from vllm.tasks import GenerationTask, PoolingTask, SupportedTask
from vllm.tracing import instrument
from vllm.utils import length_from_prompt_token_ids_or_embeds
from vllm.utils.gc_utils import freeze_gc_for_cudagraph_capture
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
from vllm.utils.math_utils import cdiv, round_up
from vllm.utils.mem_utils import DeviceMemoryProfiler, format_gib
@@ -6614,29 +6615,7 @@ class GPUModelRunner(
logger.debug("Initialized minimal KV cache for CUDA graph profiling")
@staticmethod
@contextmanager
def _freeze_gc():
gc_was_enabled = gc.isenabled()
gc.collect()
should_freeze = not envs.VLLM_ENABLE_CUDAGRAPH_GC
if should_freeze:
gc.freeze()
# A Triton kernel finalized during stream capture unloads its
# module and invalidates the captured graph.
gc.disable()
try:
yield
finally:
if should_freeze:
try:
gc.unfreeze()
gc.collect()
finally:
if gc_was_enabled:
gc.enable()
else:
gc.disable()
_freeze_gc = staticmethod(freeze_gc_for_cudagraph_capture)
def shutdown(self) -> None:
"""Release GPU tensors (model weights, KV caches, workspace) so that