[Core][MRV2] Freeze gc during V2 CG capture; skip per-descriptor cleanup (#54646)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -365,15 +365,13 @@ class BreakableCUDAGraphWrapper:
|
||||
else:
|
||||
set_graph_pool_id(current_platform.graph_pool_handle())
|
||||
|
||||
# Match torch.cuda.graph()'s pre-capture cleanup once per descriptor.
|
||||
# We drive capture_begin/end directly and bypass torch.cuda.graph(),
|
||||
# so its built-in gc + empty_cache never fire. Run them here once
|
||||
# per _capture call -- NOT inside _begin_segment, since this capture
|
||||
# session may issue many begin/end pairs (one per layer's break),
|
||||
# and repeated gc would tank capture time the way it did for the
|
||||
# pre-`gc_disable` piecewise path.
|
||||
gc.collect()
|
||||
torch.accelerator.empty_cache()
|
||||
# Match torch.cuda.graph()'s pre-capture cleanup, which we bypass.
|
||||
# Skip it when gc is disabled: bulk capture runs under
|
||||
# freeze_gc_for_cudagraph_capture, which already did this cleanup,
|
||||
# and repeating it per descriptor dominates capture time.
|
||||
if gc.isenabled():
|
||||
gc.collect()
|
||||
torch.accelerator.empty_cache()
|
||||
# Sync the offloader's copy stream before capture so any in-flight
|
||||
# pre-capture prefetches are complete and don't leak into the graph.
|
||||
get_offloader().sync_prev_onload()
|
||||
|
||||
+29
-1
@@ -4,7 +4,7 @@ import gc
|
||||
import json
|
||||
import time
|
||||
from collections import Counter
|
||||
from contextlib import suppress
|
||||
from contextlib import contextmanager, suppress
|
||||
from typing import Any
|
||||
|
||||
import vllm.envs as envs
|
||||
@@ -93,6 +93,34 @@ class GCDebugger:
|
||||
)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def freeze_gc_for_cudagraph_capture():
|
||||
"""Freeze and disable gc for the duration of bulk CUDA graph capture.
|
||||
|
||||
A gc cycle during stream capture can invalidate the captured graph, e.g.
|
||||
a finalized Triton kernel unloads its module. Opt out with
|
||||
VLLM_ENABLE_CUDAGRAPH_GC=1.
|
||||
"""
|
||||
gc_was_enabled = gc.isenabled()
|
||||
gc.collect()
|
||||
should_freeze = not envs.VLLM_ENABLE_CUDAGRAPH_GC
|
||||
if should_freeze:
|
||||
gc.freeze()
|
||||
gc.disable()
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
if should_freeze:
|
||||
try:
|
||||
gc.unfreeze()
|
||||
gc.collect()
|
||||
finally:
|
||||
if gc_was_enabled:
|
||||
gc.enable()
|
||||
else:
|
||||
gc.disable()
|
||||
|
||||
|
||||
def freeze_gc_heap() -> None:
|
||||
"""
|
||||
Freeze all objects tracked by the garbage collector. It should be invoked
|
||||
|
||||
@@ -61,6 +61,7 @@ from vllm.multimodal.encoder_budget import (
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.tasks import SupportedTask
|
||||
from vllm.utils.gc_utils import freeze_gc_for_cudagraph_capture
|
||||
from vllm.utils.mem_utils import DeviceMemoryProfiler, format_gib
|
||||
from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE
|
||||
from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput
|
||||
@@ -916,44 +917,47 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
compilation_counter.num_gpu_runner_capture_triggers += 1
|
||||
|
||||
start_time = time.perf_counter()
|
||||
gc.collect()
|
||||
torch.accelerator.empty_cache()
|
||||
start_free_gpu_memory = torch.accelerator.get_memory_info()[0]
|
||||
with freeze_gc_for_cudagraph_capture():
|
||||
torch.accelerator.empty_cache()
|
||||
start_free_gpu_memory = torch.accelerator.get_memory_info()[0]
|
||||
|
||||
with self.maybe_setup_dummy_loras(self.lora_config):
|
||||
if capture_encoder:
|
||||
self.model_state.encoder_runner.capture()
|
||||
with self.maybe_setup_dummy_loras(self.lora_config):
|
||||
if capture_encoder:
|
||||
self.model_state.encoder_runner.capture()
|
||||
|
||||
if capture_decoder:
|
||||
input_buffers = self.input_buffers
|
||||
if self.pcp_manager is not None:
|
||||
input_buffers = self.pcp_manager.input_buffers
|
||||
self.cudagraph_manager.capture(
|
||||
self.model,
|
||||
self.model_state,
|
||||
input_buffers,
|
||||
self.intermediate_tensors,
|
||||
self.block_tables,
|
||||
self.attn_groups,
|
||||
self.kv_cache_config,
|
||||
pcp_manager=self.pcp_manager,
|
||||
has_lora=self.lora_config is not None,
|
||||
use_aux_hidden_state_outputs=self.use_aux_hidden_state_outputs,
|
||||
lora_capture_hook=create_lora_capture_hook(self.lora_config, self),
|
||||
)
|
||||
if self.speculator is not None:
|
||||
with use_workspace_lane(self._draft_workspace_lane):
|
||||
self.speculator.capture()
|
||||
if self.adaptive_verification is not None:
|
||||
with self.step_timing.collect() as timings:
|
||||
for batch in self.adaptive_verification.batches_to_profile(
|
||||
self.cudagraph_manager.captured_token_counts()
|
||||
):
|
||||
self._dummy_run(**batch)
|
||||
self.adaptive_verification.set_initial_cost_curves(timings)
|
||||
if capture_decoder:
|
||||
input_buffers = self.input_buffers
|
||||
if self.pcp_manager is not None:
|
||||
input_buffers = self.pcp_manager.input_buffers
|
||||
self.cudagraph_manager.capture(
|
||||
self.model,
|
||||
self.model_state,
|
||||
input_buffers,
|
||||
self.intermediate_tensors,
|
||||
self.block_tables,
|
||||
self.attn_groups,
|
||||
self.kv_cache_config,
|
||||
pcp_manager=self.pcp_manager,
|
||||
has_lora=self.lora_config is not None,
|
||||
use_aux_hidden_state_outputs=self.use_aux_hidden_state_outputs,
|
||||
lora_capture_hook=create_lora_capture_hook(
|
||||
self.lora_config, self
|
||||
),
|
||||
)
|
||||
if self.speculator is not None:
|
||||
with use_workspace_lane(self._draft_workspace_lane):
|
||||
self.speculator.capture()
|
||||
if self.adaptive_verification is not None:
|
||||
with self.step_timing.collect() as timings:
|
||||
for batch in self.adaptive_verification.batches_to_profile(
|
||||
self.cudagraph_manager.captured_token_counts()
|
||||
):
|
||||
self._dummy_run(**batch)
|
||||
self.adaptive_verification.set_initial_cost_curves(timings)
|
||||
|
||||
end_free_gpu_memory = torch.accelerator.get_memory_info()[0]
|
||||
|
||||
end_time = time.perf_counter()
|
||||
end_free_gpu_memory = torch.accelerator.get_memory_info()[0]
|
||||
elapsed_time = end_time - start_time
|
||||
cuda_graph_size = start_free_gpu_memory - end_free_gpu_memory
|
||||
# This usually takes 5~20 seconds.
|
||||
|
||||
@@ -124,6 +124,7 @@ from vllm.sequence import IntermediateTensors
|
||||
from vllm.tasks import GenerationTask, PoolingTask, SupportedTask
|
||||
from vllm.tracing import instrument
|
||||
from vllm.utils import length_from_prompt_token_ids_or_embeds
|
||||
from vllm.utils.gc_utils import freeze_gc_for_cudagraph_capture
|
||||
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
||||
from vllm.utils.math_utils import cdiv, round_up
|
||||
from vllm.utils.mem_utils import DeviceMemoryProfiler, format_gib
|
||||
@@ -6614,29 +6615,7 @@ class GPUModelRunner(
|
||||
|
||||
logger.debug("Initialized minimal KV cache for CUDA graph profiling")
|
||||
|
||||
@staticmethod
|
||||
@contextmanager
|
||||
def _freeze_gc():
|
||||
gc_was_enabled = gc.isenabled()
|
||||
gc.collect()
|
||||
should_freeze = not envs.VLLM_ENABLE_CUDAGRAPH_GC
|
||||
if should_freeze:
|
||||
gc.freeze()
|
||||
# A Triton kernel finalized during stream capture unloads its
|
||||
# module and invalidates the captured graph.
|
||||
gc.disable()
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
if should_freeze:
|
||||
try:
|
||||
gc.unfreeze()
|
||||
gc.collect()
|
||||
finally:
|
||||
if gc_was_enabled:
|
||||
gc.enable()
|
||||
else:
|
||||
gc.disable()
|
||||
_freeze_gc = staticmethod(freeze_gc_for_cudagraph_capture)
|
||||
|
||||
def shutdown(self) -> None:
|
||||
"""Release GPU tensors (model weights, KV caches, workspace) so that
|
||||
|
||||
Reference in New Issue
Block a user