diff --git a/vllm/compilation/breakable_cudagraph.py b/vllm/compilation/breakable_cudagraph.py index 6da3ec7178..343b9ef398 100644 --- a/vllm/compilation/breakable_cudagraph.py +++ b/vllm/compilation/breakable_cudagraph.py @@ -365,15 +365,13 @@ class BreakableCUDAGraphWrapper: else: set_graph_pool_id(current_platform.graph_pool_handle()) - # Match torch.cuda.graph()'s pre-capture cleanup once per descriptor. - # We drive capture_begin/end directly and bypass torch.cuda.graph(), - # so its built-in gc + empty_cache never fire. Run them here once - # per _capture call -- NOT inside _begin_segment, since this capture - # session may issue many begin/end pairs (one per layer's break), - # and repeated gc would tank capture time the way it did for the - # pre-`gc_disable` piecewise path. - gc.collect() - torch.accelerator.empty_cache() + # Match torch.cuda.graph()'s pre-capture cleanup, which we bypass. + # Skip it when gc is disabled: bulk capture runs under + # freeze_gc_for_cudagraph_capture, which already did this cleanup, + # and repeating it per descriptor dominates capture time. + if gc.isenabled(): + gc.collect() + torch.accelerator.empty_cache() # Sync the offloader's copy stream before capture so any in-flight # pre-capture prefetches are complete and don't leak into the graph. get_offloader().sync_prev_onload() diff --git a/vllm/utils/gc_utils.py b/vllm/utils/gc_utils.py index 25c8653e0e..32bdfb7648 100644 --- a/vllm/utils/gc_utils.py +++ b/vllm/utils/gc_utils.py @@ -4,7 +4,7 @@ import gc import json import time from collections import Counter -from contextlib import suppress +from contextlib import contextmanager, suppress from typing import Any import vllm.envs as envs @@ -93,6 +93,34 @@ class GCDebugger: ) +@contextmanager +def freeze_gc_for_cudagraph_capture(): + """Freeze and disable gc for the duration of bulk CUDA graph capture. + + A gc cycle during stream capture can invalidate the captured graph, e.g. + a finalized Triton kernel unloads its module. Opt out with + VLLM_ENABLE_CUDAGRAPH_GC=1. + """ + gc_was_enabled = gc.isenabled() + gc.collect() + should_freeze = not envs.VLLM_ENABLE_CUDAGRAPH_GC + if should_freeze: + gc.freeze() + gc.disable() + try: + yield + finally: + if should_freeze: + try: + gc.unfreeze() + gc.collect() + finally: + if gc_was_enabled: + gc.enable() + else: + gc.disable() + + def freeze_gc_heap() -> None: """ Freeze all objects tracked by the garbage collector. It should be invoked diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index 89ac0aebf8..a7db9a85b4 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -61,6 +61,7 @@ from vllm.multimodal.encoder_budget import ( ) from vllm.sequence import IntermediateTensors from vllm.tasks import SupportedTask +from vllm.utils.gc_utils import freeze_gc_for_cudagraph_capture from vllm.utils.mem_utils import DeviceMemoryProfiler, format_gib from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput @@ -916,44 +917,47 @@ class GPUModelRunner(LoRAModelRunnerMixin): compilation_counter.num_gpu_runner_capture_triggers += 1 start_time = time.perf_counter() - gc.collect() - torch.accelerator.empty_cache() - start_free_gpu_memory = torch.accelerator.get_memory_info()[0] + with freeze_gc_for_cudagraph_capture(): + torch.accelerator.empty_cache() + start_free_gpu_memory = torch.accelerator.get_memory_info()[0] - with self.maybe_setup_dummy_loras(self.lora_config): - if capture_encoder: - self.model_state.encoder_runner.capture() + with self.maybe_setup_dummy_loras(self.lora_config): + if capture_encoder: + self.model_state.encoder_runner.capture() - if capture_decoder: - input_buffers = self.input_buffers - if self.pcp_manager is not None: - input_buffers = self.pcp_manager.input_buffers - self.cudagraph_manager.capture( - self.model, - self.model_state, - input_buffers, - self.intermediate_tensors, - self.block_tables, - self.attn_groups, - self.kv_cache_config, - pcp_manager=self.pcp_manager, - has_lora=self.lora_config is not None, - use_aux_hidden_state_outputs=self.use_aux_hidden_state_outputs, - lora_capture_hook=create_lora_capture_hook(self.lora_config, self), - ) - if self.speculator is not None: - with use_workspace_lane(self._draft_workspace_lane): - self.speculator.capture() - if self.adaptive_verification is not None: - with self.step_timing.collect() as timings: - for batch in self.adaptive_verification.batches_to_profile( - self.cudagraph_manager.captured_token_counts() - ): - self._dummy_run(**batch) - self.adaptive_verification.set_initial_cost_curves(timings) + if capture_decoder: + input_buffers = self.input_buffers + if self.pcp_manager is not None: + input_buffers = self.pcp_manager.input_buffers + self.cudagraph_manager.capture( + self.model, + self.model_state, + input_buffers, + self.intermediate_tensors, + self.block_tables, + self.attn_groups, + self.kv_cache_config, + pcp_manager=self.pcp_manager, + has_lora=self.lora_config is not None, + use_aux_hidden_state_outputs=self.use_aux_hidden_state_outputs, + lora_capture_hook=create_lora_capture_hook( + self.lora_config, self + ), + ) + if self.speculator is not None: + with use_workspace_lane(self._draft_workspace_lane): + self.speculator.capture() + if self.adaptive_verification is not None: + with self.step_timing.collect() as timings: + for batch in self.adaptive_verification.batches_to_profile( + self.cudagraph_manager.captured_token_counts() + ): + self._dummy_run(**batch) + self.adaptive_verification.set_initial_cost_curves(timings) + + end_free_gpu_memory = torch.accelerator.get_memory_info()[0] end_time = time.perf_counter() - end_free_gpu_memory = torch.accelerator.get_memory_info()[0] elapsed_time = end_time - start_time cuda_graph_size = start_free_gpu_memory - end_free_gpu_memory # This usually takes 5~20 seconds. diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py index e6ba7bca77..fdab24df0e 100644 --- a/vllm/v1/worker/gpu_model_runner.py +++ b/vllm/v1/worker/gpu_model_runner.py @@ -124,6 +124,7 @@ from vllm.sequence import IntermediateTensors from vllm.tasks import GenerationTask, PoolingTask, SupportedTask from vllm.tracing import instrument from vllm.utils import length_from_prompt_token_ids_or_embeds +from vllm.utils.gc_utils import freeze_gc_for_cudagraph_capture from vllm.utils.gpu_sync_debug import gpu_sync_allowed from vllm.utils.math_utils import cdiv, round_up from vllm.utils.mem_utils import DeviceMemoryProfiler, format_gib @@ -6614,29 +6615,7 @@ class GPUModelRunner( logger.debug("Initialized minimal KV cache for CUDA graph profiling") - @staticmethod - @contextmanager - def _freeze_gc(): - gc_was_enabled = gc.isenabled() - gc.collect() - should_freeze = not envs.VLLM_ENABLE_CUDAGRAPH_GC - if should_freeze: - gc.freeze() - # A Triton kernel finalized during stream capture unloads its - # module and invalidates the captured graph. - gc.disable() - try: - yield - finally: - if should_freeze: - try: - gc.unfreeze() - gc.collect() - finally: - if gc_was_enabled: - gc.enable() - else: - gc.disable() + _freeze_gc = staticmethod(freeze_gc_for_cudagraph_capture) def shutdown(self) -> None: """Release GPU tensors (model weights, KV caches, workspace) so that